Add StereoCodecAdapter classes
This CL is the step 1 for adding alpha channel support over the wire in webrtc.
- Add the footprint for adapter classes that wraps actual codecs.
- This CL does not add a webrtc::VideoFrame container that can carry alpha to
make the CL shorter for an easier review. Therefore, it exercises a code path
for when we receive no alpha input, just regular I420 frames.
- Unittest sends a video frame for encode/decode through these adapters and
checks the output PSNR.
- See https://webrtc-review.googlesource.com/c/src/+/7800 for the experimental
CL that gives an idea about how it will come together.
Design Doc: https://goo.gl/sFeSUT
Bug: webrtc:7671
Change-Id: I9d3be13647a0a958feceb8d7a9aa93852fc6a1fa
Reviewed-on: https://webrtc-review.googlesource.com/11841
Commit-Queue: Emircan Uysaler <emircan@webrtc.org>
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Reviewed-by: Niklas Enbom <niklas.enbom@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#20490}diff --git a/api/BUILD.gn b/api/BUILD.gn
index 581f1c2..72078b4 100644
--- a/api/BUILD.gn
+++ b/api/BUILD.gn
@@ -304,6 +304,23 @@
]
}
+ rtc_source_set("mock_video_codec_factory") {
+ testonly = true
+ sources = [
+ "test/mock_video_decoder_factory.h",
+ "test/mock_video_encoder_factory.h",
+ ]
+
+ public_deps = [
+ "../api/video_codecs:video_codecs_api",
+ ]
+
+ deps = [
+ "../test:test_support",
+ "//testing/gmock",
+ ]
+ }
+
rtc_source_set("fakemetricsobserver") {
testonly = true
sources = [
diff --git a/api/test/mock_video_decoder_factory.h b/api/test/mock_video_decoder_factory.h
new file mode 100644
index 0000000..915e391
--- /dev/null
+++ b/api/test/mock_video_decoder_factory.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_TEST_MOCK_VIDEO_DECODER_FACTORY_H_
+#define API_TEST_MOCK_VIDEO_DECODER_FACTORY_H_
+
+#include <memory>
+#include <vector>
+
+#include "api/video_codecs/sdp_video_format.h"
+#include "api/video_codecs/video_decoder_factory.h"
+#include "test/gmock.h"
+
+namespace webrtc {
+
+class MockVideoDecoderFactory : public webrtc::VideoDecoderFactory {
+ public:
+ MOCK_CONST_METHOD0(GetSupportedFormats,
+ std::vector<webrtc::SdpVideoFormat>());
+
+ // We need to proxy to a return type that is copyable.
+ std::unique_ptr<webrtc::VideoDecoder> CreateVideoDecoder(
+ const webrtc::SdpVideoFormat& format) {
+ return std::unique_ptr<webrtc::VideoDecoder>(
+ CreateVideoDecoderProxy(format));
+ }
+ MOCK_METHOD1(CreateVideoDecoderProxy,
+ webrtc::VideoDecoder*(const webrtc::SdpVideoFormat&));
+
+ MOCK_METHOD0(Die, void());
+ ~MockVideoDecoderFactory() { Die(); }
+};
+} // namespace webrtc
+
+#endif // API_TEST_MOCK_VIDEO_DECODER_FACTORY_H_
diff --git a/api/test/mock_video_encoder_factory.h b/api/test/mock_video_encoder_factory.h
new file mode 100644
index 0000000..a694b63
--- /dev/null
+++ b/api/test/mock_video_encoder_factory.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_TEST_MOCK_VIDEO_ENCODER_FACTORY_H_
+#define API_TEST_MOCK_VIDEO_ENCODER_FACTORY_H_
+
+#include <memory>
+#include <vector>
+
+#include "api/video_codecs/sdp_video_format.h"
+#include "api/video_codecs/video_encoder_factory.h"
+#include "test/gmock.h"
+
+namespace webrtc {
+
+class MockVideoEncoderFactory : public webrtc::VideoEncoderFactory {
+ public:
+ MOCK_CONST_METHOD0(GetSupportedFormats,
+ std::vector<webrtc::SdpVideoFormat>());
+ MOCK_CONST_METHOD1(QueryVideoEncoder,
+ CodecInfo(const webrtc::SdpVideoFormat&));
+
+ // We need to proxy to a return type that is copyable.
+ std::unique_ptr<webrtc::VideoEncoder> CreateVideoEncoder(
+ const webrtc::SdpVideoFormat& format) {
+ return std::unique_ptr<webrtc::VideoEncoder>(
+ CreateVideoEncoderProxy(format));
+ }
+ MOCK_METHOD1(CreateVideoEncoderProxy,
+ webrtc::VideoEncoder*(const webrtc::SdpVideoFormat&));
+
+ MOCK_METHOD0(Die, void());
+ ~MockVideoEncoderFactory() { Die(); }
+};
+
+} // namespace webrtc
+
+#endif // API_TEST_MOCK_VIDEO_ENCODER_FACTORY_H_
diff --git a/media/BUILD.gn b/media/BUILD.gn
index e98001a..2e46ee5 100644
--- a/media/BUILD.gn
+++ b/media/BUILD.gn
@@ -523,6 +523,7 @@
":rtc_media",
":rtc_media_base",
":rtc_media_tests_utils",
+ "../api:mock_video_codec_factory",
"../api:video_frame_api",
"../api/audio_codecs:builtin_audio_decoder_factory",
"../api/audio_codecs:builtin_audio_encoder_factory",
diff --git a/media/engine/webrtcvideoengine_unittest.cc b/media/engine/webrtcvideoengine_unittest.cc
index 7fc5924..c3d3446 100644
--- a/media/engine/webrtcvideoengine_unittest.cc
+++ b/media/engine/webrtcvideoengine_unittest.cc
@@ -13,6 +13,8 @@
#include <memory>
#include <vector>
+#include "api/test/mock_video_decoder_factory.h"
+#include "api/test/mock_video_encoder_factory.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_decoder_factory.h"
#include "api/video_codecs/video_encoder.h"
@@ -863,44 +865,6 @@
ASSERT_EQ(1u, decoder_factory_->decoders().size());
}
-class MockVideoEncoderFactory : public webrtc::VideoEncoderFactory {
- public:
- MOCK_CONST_METHOD0(GetSupportedFormats,
- std::vector<webrtc::SdpVideoFormat>());
- MOCK_CONST_METHOD1(QueryVideoEncoder,
- CodecInfo(const webrtc::SdpVideoFormat&));
-
- // We need to proxy to a return type that is copyable.
- std::unique_ptr<webrtc::VideoEncoder> CreateVideoEncoder(
- const webrtc::SdpVideoFormat& format) {
- return std::unique_ptr<webrtc::VideoEncoder>(
- CreateVideoEncoderProxy(format));
- }
- MOCK_METHOD1(CreateVideoEncoderProxy,
- webrtc::VideoEncoder*(const webrtc::SdpVideoFormat&));
-
- MOCK_METHOD0(Die, void());
- ~MockVideoEncoderFactory() { Die(); }
-};
-
-class MockVideoDecoderFactory : public webrtc::VideoDecoderFactory {
- public:
- MOCK_CONST_METHOD0(GetSupportedFormats,
- std::vector<webrtc::SdpVideoFormat>());
-
- // We need to proxy to a return type that is copyable.
- std::unique_ptr<webrtc::VideoDecoder> CreateVideoDecoder(
- const webrtc::SdpVideoFormat& format) {
- return std::unique_ptr<webrtc::VideoDecoder>(
- CreateVideoDecoderProxy(format));
- }
- MOCK_METHOD1(CreateVideoDecoderProxy,
- webrtc::VideoDecoder*(const webrtc::SdpVideoFormat&));
-
- MOCK_METHOD0(Die, void());
- ~MockVideoDecoderFactory() { Die(); }
-};
-
TEST(WebRtcVideoEngineNewVideoCodecFactoryTest, NullFactories) {
std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory;
std::unique_ptr<webrtc::VideoDecoderFactory> decoder_factory;
@@ -911,8 +875,10 @@
TEST(WebRtcVideoEngineNewVideoCodecFactoryTest, EmptyFactories) {
// |engine| take ownership of the factories.
- MockVideoEncoderFactory* encoder_factory = new MockVideoEncoderFactory();
- MockVideoDecoderFactory* decoder_factory = new MockVideoDecoderFactory();
+ webrtc::MockVideoEncoderFactory* encoder_factory =
+ new webrtc::MockVideoEncoderFactory();
+ webrtc::MockVideoDecoderFactory* decoder_factory =
+ new webrtc::MockVideoDecoderFactory();
WebRtcVideoEngine engine(
(std::unique_ptr<webrtc::VideoEncoderFactory>(encoder_factory)),
(std::unique_ptr<webrtc::VideoDecoderFactory>(decoder_factory)));
@@ -928,8 +894,10 @@
// new factories.
TEST(WebRtcVideoEngineNewVideoCodecFactoryTest, Vp8) {
// |engine| take ownership of the factories.
- MockVideoEncoderFactory* encoder_factory = new MockVideoEncoderFactory();
- MockVideoDecoderFactory* decoder_factory = new MockVideoDecoderFactory();
+ webrtc::MockVideoEncoderFactory* encoder_factory =
+ new webrtc::MockVideoEncoderFactory();
+ webrtc::MockVideoDecoderFactory* decoder_factory =
+ new webrtc::MockVideoDecoderFactory();
WebRtcVideoEngine engine(
(std::unique_ptr<webrtc::VideoEncoderFactory>(encoder_factory)),
(std::unique_ptr<webrtc::VideoDecoderFactory>(decoder_factory)));
diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index eb0a419..d1e3311 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -92,6 +92,7 @@
":video_coding_utility",
":webrtc_h264",
":webrtc_i420",
+ ":webrtc_stereo",
":webrtc_vp8",
":webrtc_vp9",
"..:module_api",
@@ -233,6 +234,30 @@
]
}
+rtc_static_library("webrtc_stereo") {
+ sources = [
+ "codecs/stereo/include/stereo_decoder_adapter.h",
+ "codecs/stereo/include/stereo_encoder_adapter.h",
+ "codecs/stereo/stereo_decoder_adapter.cc",
+ "codecs/stereo/stereo_encoder_adapter.cc",
+ ]
+
+ if (!build_with_chromium && is_clang) {
+ # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163).
+ suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
+ }
+
+ deps = [
+ ":video_coding_utility",
+ "..:module_api",
+ "../..:webrtc_common",
+ "../../api/video_codecs:video_codecs_api",
+ "../../common_video:common_video",
+ "../../rtc_base:rtc_base_approved",
+ "../../system_wrappers",
+ ]
+}
+
rtc_static_library("webrtc_vp8") {
sources = [
"codecs/vp8/default_temporal_layers.cc",
@@ -423,6 +448,7 @@
}
sources = [
"codecs/h264/test/h264_impl_unittest.cc",
+ "codecs/stereo/test/stereo_adapter_unittest.cc",
"codecs/test/videoprocessor_integrationtest.cc",
"codecs/test/videoprocessor_integrationtest.h",
"codecs/test/videoprocessor_integrationtest_libvpx.cc",
@@ -437,9 +463,11 @@
":video_coding",
":video_coding_utility",
":webrtc_h264",
+ ":webrtc_stereo",
":webrtc_vp8",
":webrtc_vp9",
"../..:webrtc_common",
+ "../../api:mock_video_codec_factory",
"../../api:optional",
"../../api:video_frame_api",
"../../common_video",
diff --git a/modules/video_coding/codecs/stereo/OWNERS b/modules/video_coding/codecs/stereo/OWNERS
new file mode 100644
index 0000000..6b72be3
--- /dev/null
+++ b/modules/video_coding/codecs/stereo/OWNERS
@@ -0,0 +1 @@
+emircan@webrtc.org
diff --git a/modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h b/modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h
new file mode 100644
index 0000000..729517f
--- /dev/null
+++ b/modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_CODECS_STEREO_INCLUDE_STEREO_DECODER_ADAPTER_H_
+#define MODULES_VIDEO_CODING_CODECS_STEREO_INCLUDE_STEREO_DECODER_ADAPTER_H_
+
+#include <map>
+#include <memory>
+#include <vector>
+
+#include "api/video_codecs/video_decoder.h"
+#include "api/video_codecs/video_decoder_factory.h"
+#include "modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h"
+
+namespace webrtc {
+
+class StereoDecoderAdapter : public VideoDecoder {
+ public:
+ // |factory| is not owned and expected to outlive this class' lifetime.
+ explicit StereoDecoderAdapter(VideoDecoderFactory* factory);
+ virtual ~StereoDecoderAdapter();
+
+ // Implements VideoDecoder
+ int32_t InitDecode(const VideoCodec* codec_settings,
+ int32_t number_of_cores) override;
+ int32_t Decode(const EncodedImage& input_image,
+ bool missing_frames,
+ const RTPFragmentationHeader* fragmentation,
+ const CodecSpecificInfo* codec_specific_info,
+ int64_t render_time_ms) override;
+ int32_t RegisterDecodeCompleteCallback(
+ DecodedImageCallback* callback) override;
+ int32_t Release() override;
+
+ void Decoded(AlphaCodecStream stream_idx,
+ VideoFrame* decoded_image,
+ rtc::Optional<int32_t> decode_time_ms,
+ rtc::Optional<uint8_t> qp);
+
+ private:
+ // Wrapper class that redirects Decoded() calls.
+ class AdapterDecodedImageCallback;
+
+ // Holds the decoded image output of a frame.
+ struct DecodedImageData;
+
+ void MergeAlphaImages(VideoFrame* decoded_image,
+ const rtc::Optional<int32_t>& decode_time_ms,
+ const rtc::Optional<uint8_t>& qp,
+ VideoFrame* stereo_decoded_image,
+ const rtc::Optional<int32_t>& stereo_decode_time_ms,
+ const rtc::Optional<uint8_t>& stereo_qp);
+
+ VideoDecoderFactory* const factory_;
+ std::vector<std::unique_ptr<VideoDecoder>> decoders_;
+ std::vector<std::unique_ptr<AdapterDecodedImageCallback>> adapter_callbacks_;
+ DecodedImageCallback* decoded_complete_callback_;
+
+ // Holds YUV or AXX decode output of a frame that is identified by timestamp.
+ std::map<uint32_t /* timestamp */, DecodedImageData> decoded_data_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_CODECS_STEREO_INCLUDE_STEREO_DECODER_ADAPTER_H_
diff --git a/modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h b/modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h
new file mode 100644
index 0000000..ef1e9e1
--- /dev/null
+++ b/modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_CODECS_STEREO_INCLUDE_STEREO_ENCODER_ADAPTER_H_
+#define MODULES_VIDEO_CODING_CODECS_STEREO_INCLUDE_STEREO_ENCODER_ADAPTER_H_
+
+#include <memory>
+#include <vector>
+
+#include "api/video_codecs/video_encoder.h"
+#include "api/video_codecs/video_encoder_factory.h"
+#include "modules/video_coding/include/video_codec_interface.h"
+
+namespace webrtc {
+
+enum AlphaCodecStream {
+ kYUVStream = 0,
+ kAXXStream = 1,
+ kAlphaCodecStreams = 2,
+};
+
+class StereoEncoderAdapter : public VideoEncoder {
+ public:
+ // |factory| is not owned and expected to outlive this class' lifetime.
+ explicit StereoEncoderAdapter(VideoEncoderFactory* factory);
+ virtual ~StereoEncoderAdapter();
+
+ // Implements VideoEncoder
+ int InitEncode(const VideoCodec* inst,
+ int number_of_cores,
+ size_t max_payload_size) override;
+ int Encode(const VideoFrame& input_image,
+ const CodecSpecificInfo* codec_specific_info,
+ const std::vector<FrameType>* frame_types) override;
+ int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override;
+ int SetChannelParameters(uint32_t packet_loss, int64_t rtt) override;
+ int SetRateAllocation(const BitrateAllocation& bitrate,
+ uint32_t new_framerate) override;
+ int Release() override;
+ const char* ImplementationName() const override;
+
+ EncodedImageCallback::Result OnEncodedImage(
+ AlphaCodecStream stream_idx,
+ const EncodedImage& encodedImage,
+ const CodecSpecificInfo* codecSpecificInfo,
+ const RTPFragmentationHeader* fragmentation);
+
+ private:
+ // Wrapper class that redirects OnEncodedImage() calls.
+ class AdapterEncodedImageCallback;
+
+ // Holds the encoded image output of a frame.
+ struct EncodedImageData;
+
+ VideoEncoderFactory* const factory_;
+ std::vector<std::unique_ptr<VideoEncoder>> encoders_;
+ std::vector<std::unique_ptr<AdapterEncodedImageCallback>> adapter_callbacks_;
+ EncodedImageCallback* encoded_complete_callback_;
+
+ uint64_t picture_index_ = 0;
+ std::vector<uint8_t> stereo_dummy_planes_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_CODECS_STEREO_INCLUDE_STEREO_ENCODER_ADAPTER_H_
diff --git a/modules/video_coding/codecs/stereo/stereo_decoder_adapter.cc b/modules/video_coding/codecs/stereo/stereo_decoder_adapter.cc
new file mode 100644
index 0000000..5f8b4c59
--- /dev/null
+++ b/modules/video_coding/codecs/stereo/stereo_decoder_adapter.cc
@@ -0,0 +1,183 @@
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h"
+
+#include "api/video/i420_buffer.h"
+#include "api/video_codecs/sdp_video_format.h"
+#include "common_video/include/video_frame.h"
+#include "common_video/include/video_frame_buffer.h"
+#include "common_video/libyuv/include/webrtc_libyuv.h"
+#include "rtc_base/keep_ref_until_done.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+class StereoDecoderAdapter::AdapterDecodedImageCallback
+ : public webrtc::DecodedImageCallback {
+ public:
+ AdapterDecodedImageCallback(webrtc::StereoDecoderAdapter* adapter,
+ AlphaCodecStream stream_idx)
+ : adapter_(adapter), stream_idx_(stream_idx) {}
+
+ void Decoded(VideoFrame& decodedImage,
+ rtc::Optional<int32_t> decode_time_ms,
+ rtc::Optional<uint8_t> qp) override {
+ if (!adapter_)
+ return;
+ adapter_->Decoded(stream_idx_, &decodedImage, decode_time_ms, qp);
+ }
+ int32_t Decoded(VideoFrame& decodedImage) override {
+ RTC_NOTREACHED();
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
+ int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override {
+ RTC_NOTREACHED();
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
+
+ private:
+ StereoDecoderAdapter* adapter_;
+ const AlphaCodecStream stream_idx_;
+};
+
+struct StereoDecoderAdapter::DecodedImageData {
+ explicit DecodedImageData(AlphaCodecStream stream_idx)
+ : stream_idx_(stream_idx),
+ decodedImage_(I420Buffer::Create(1 /* width */, 1 /* height */),
+ 0,
+ 0,
+ kVideoRotation_0) {
+ RTC_DCHECK_EQ(kAXXStream, stream_idx);
+ }
+ DecodedImageData(AlphaCodecStream stream_idx,
+ const VideoFrame& decodedImage,
+ const rtc::Optional<int32_t>& decode_time_ms,
+ const rtc::Optional<uint8_t>& qp)
+ : stream_idx_(stream_idx),
+ decodedImage_(decodedImage),
+ decode_time_ms_(decode_time_ms),
+ qp_(qp) {}
+ const AlphaCodecStream stream_idx_;
+ VideoFrame decodedImage_;
+ const rtc::Optional<int32_t> decode_time_ms_;
+ const rtc::Optional<uint8_t> qp_;
+
+ private:
+ RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(DecodedImageData);
+};
+
+StereoDecoderAdapter::StereoDecoderAdapter(VideoDecoderFactory* factory)
+ : factory_(factory) {}
+
+StereoDecoderAdapter::~StereoDecoderAdapter() {
+ Release();
+}
+
+int32_t StereoDecoderAdapter::InitDecode(const VideoCodec* codec_settings,
+ int32_t number_of_cores) {
+ VideoCodec settings = *codec_settings;
+ settings.codecType = kVideoCodecVP9;
+ for (size_t i = 0; i < kAlphaCodecStreams; ++i) {
+ const SdpVideoFormat format("VP9");
+ std::unique_ptr<VideoDecoder> decoder =
+ factory_->CreateVideoDecoder(format);
+ const int32_t rv = decoder->InitDecode(&settings, number_of_cores);
+ if (rv)
+ return rv;
+ adapter_callbacks_.emplace_back(
+ new StereoDecoderAdapter::AdapterDecodedImageCallback(
+ this, static_cast<AlphaCodecStream>(i)));
+ decoder->RegisterDecodeCompleteCallback(adapter_callbacks_.back().get());
+ decoders_.emplace_back(std::move(decoder));
+ }
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t StereoDecoderAdapter::Decode(
+ const EncodedImage& input_image,
+ bool missing_frames,
+ const RTPFragmentationHeader* /*fragmentation*/,
+ const CodecSpecificInfo* codec_specific_info,
+ int64_t render_time_ms) {
+ // TODO(emircan): Read |codec_specific_info->stereoInfo| to split frames.
+ int32_t rv =
+ decoders_[kYUVStream]->Decode(input_image, missing_frames, nullptr,
+ codec_specific_info, render_time_ms);
+ if (rv)
+ return rv;
+ rv = decoders_[kAXXStream]->Decode(input_image, missing_frames, nullptr,
+ codec_specific_info, render_time_ms);
+ return rv;
+}
+
+int32_t StereoDecoderAdapter::RegisterDecodeCompleteCallback(
+ DecodedImageCallback* callback) {
+ decoded_complete_callback_ = callback;
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t StereoDecoderAdapter::Release() {
+ for (auto& decoder : decoders_) {
+ const int32_t rv = decoder->Release();
+ if (rv)
+ return rv;
+ }
+ decoders_.clear();
+ adapter_callbacks_.clear();
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+void StereoDecoderAdapter::Decoded(AlphaCodecStream stream_idx,
+ VideoFrame* decoded_image,
+ rtc::Optional<int32_t> decode_time_ms,
+ rtc::Optional<uint8_t> qp) {
+ const auto& other_decoded_data_it =
+ decoded_data_.find(decoded_image->timestamp());
+ if (other_decoded_data_it != decoded_data_.end()) {
+ auto& other_image_data = other_decoded_data_it->second;
+ if (stream_idx == kYUVStream) {
+ RTC_DCHECK_EQ(kAXXStream, other_image_data.stream_idx_);
+ MergeAlphaImages(decoded_image, decode_time_ms, qp,
+ &other_image_data.decodedImage_,
+ other_image_data.decode_time_ms_, other_image_data.qp_);
+ } else {
+ RTC_DCHECK_EQ(kYUVStream, other_image_data.stream_idx_);
+ RTC_DCHECK_EQ(kAXXStream, stream_idx);
+ MergeAlphaImages(&other_image_data.decodedImage_,
+ other_image_data.decode_time_ms_, other_image_data.qp_,
+ decoded_image, decode_time_ms, qp);
+ }
+ decoded_data_.erase(decoded_data_.begin(), other_decoded_data_it);
+ return;
+ }
+ RTC_DCHECK(decoded_data_.find(decoded_image->timestamp()) ==
+ decoded_data_.end());
+ decoded_data_.emplace(
+ std::piecewise_construct,
+ std::forward_as_tuple(decoded_image->timestamp()),
+ std::forward_as_tuple(stream_idx, *decoded_image, decode_time_ms, qp));
+}
+
+void StereoDecoderAdapter::MergeAlphaImages(
+ VideoFrame* decodedImage,
+ const rtc::Optional<int32_t>& decode_time_ms,
+ const rtc::Optional<uint8_t>& qp,
+ VideoFrame* alpha_decodedImage,
+ const rtc::Optional<int32_t>& alpha_decode_time_ms,
+ const rtc::Optional<uint8_t>& alpha_qp) {
+ // TODO(emircan): Merge the output and put in a VideoFrame container that can
+ // transport I420A.
+ decoded_complete_callback_->Decoded(*decodedImage, decode_time_ms, qp);
+ decoded_complete_callback_->Decoded(*alpha_decodedImage, alpha_decode_time_ms,
+ alpha_qp);
+}
+
+} // namespace webrtc
diff --git a/modules/video_coding/codecs/stereo/stereo_encoder_adapter.cc b/modules/video_coding/codecs/stereo/stereo_encoder_adapter.cc
new file mode 100644
index 0000000..f6a04d0
--- /dev/null
+++ b/modules/video_coding/codecs/stereo/stereo_encoder_adapter.cc
@@ -0,0 +1,167 @@
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h"
+
+#include "api/video_codecs/sdp_video_format.h"
+#include "common_video/include/video_frame.h"
+#include "common_video/include/video_frame_buffer.h"
+#include "common_video/libyuv/include/webrtc_libyuv.h"
+#include "modules/include/module_common_types.h"
+#include "rtc_base/keep_ref_until_done.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+// Callback wrapper that helps distinguish returned results from |encoders_|
+// instances.
+class StereoEncoderAdapter::AdapterEncodedImageCallback
+ : public webrtc::EncodedImageCallback {
+ public:
+ AdapterEncodedImageCallback(webrtc::StereoEncoderAdapter* adapter,
+ AlphaCodecStream stream_idx)
+ : adapter_(adapter), stream_idx_(stream_idx) {}
+
+ EncodedImageCallback::Result OnEncodedImage(
+ const EncodedImage& encoded_image,
+ const CodecSpecificInfo* codec_specific_info,
+ const RTPFragmentationHeader* fragmentation) override {
+ if (!adapter_)
+ return Result(Result::OK);
+ return adapter_->OnEncodedImage(stream_idx_, encoded_image,
+ codec_specific_info, fragmentation);
+ }
+
+ private:
+ StereoEncoderAdapter* adapter_;
+ const AlphaCodecStream stream_idx_;
+};
+
+StereoEncoderAdapter::StereoEncoderAdapter(VideoEncoderFactory* factory)
+ : factory_(factory), encoded_complete_callback_(nullptr) {}
+
+StereoEncoderAdapter::~StereoEncoderAdapter() {
+ Release();
+}
+
+int StereoEncoderAdapter::InitEncode(const VideoCodec* inst,
+ int number_of_cores,
+ size_t max_payload_size) {
+ const size_t buffer_size =
+ CalcBufferSize(VideoType::kI420, inst->width, inst->height);
+ stereo_dummy_planes_.resize(buffer_size);
+ // It is more expensive to encode 0x00, so use 0x80 instead.
+ std::fill(stereo_dummy_planes_.begin(), stereo_dummy_planes_.end(), 0x80);
+
+ for (size_t i = 0; i < kAlphaCodecStreams; ++i) {
+ const SdpVideoFormat format("VP9");
+ std::unique_ptr<VideoEncoder> encoder =
+ factory_->CreateVideoEncoder(format);
+ const int rv = encoder->InitEncode(inst, number_of_cores, max_payload_size);
+ if (rv) {
+ LOG(LS_ERROR) << "Failed to create stere codec index " << i;
+ return rv;
+ }
+ adapter_callbacks_.emplace_back(new AdapterEncodedImageCallback(
+ this, static_cast<AlphaCodecStream>(i)));
+ encoder->RegisterEncodeCompleteCallback(adapter_callbacks_.back().get());
+ encoders_.emplace_back(std::move(encoder));
+ }
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int StereoEncoderAdapter::Encode(const VideoFrame& input_image,
+ const CodecSpecificInfo* codec_specific_info,
+ const std::vector<FrameType>* frame_types) {
+ if (!encoded_complete_callback_) {
+ return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+ }
+
+ // TODO(emircan): Extract alpha and create an alpha frame with dummy planes.
+ // Since we don't have a way of transporting alpha yet, put a dummy output for
+ // alpha consisting of YXX.
+
+ // Encode AXX
+ rtc::scoped_refptr<I420BufferInterface> yuva_buffer =
+ input_image.video_frame_buffer()->ToI420();
+ rtc::scoped_refptr<WrappedI420Buffer> alpha_buffer(
+ new rtc::RefCountedObject<webrtc::WrappedI420Buffer>(
+ input_image.width(), input_image.height(), yuva_buffer->DataY(),
+ yuva_buffer->StrideY(), stereo_dummy_planes_.data(),
+ yuva_buffer->StrideU(), stereo_dummy_planes_.data(),
+ yuva_buffer->StrideV(),
+ rtc::KeepRefUntilDone(input_image.video_frame_buffer())));
+ VideoFrame alpha_image(alpha_buffer, input_image.timestamp(),
+ input_image.render_time_ms(), input_image.rotation());
+ encoders_[kAXXStream]->Encode(alpha_image, codec_specific_info, frame_types);
+
+ // Encode YUV
+ int rv = encoders_[kYUVStream]->Encode(input_image, codec_specific_info,
+ frame_types);
+ return rv;
+}
+
+int StereoEncoderAdapter::RegisterEncodeCompleteCallback(
+ EncodedImageCallback* callback) {
+ encoded_complete_callback_ = callback;
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int StereoEncoderAdapter::SetChannelParameters(uint32_t packet_loss,
+ int64_t rtt) {
+ for (auto& encoder : encoders_) {
+ const int rv = encoder->SetChannelParameters(packet_loss, rtt);
+ if (rv)
+ return rv;
+ }
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int StereoEncoderAdapter::SetRateAllocation(const BitrateAllocation& bitrate,
+ uint32_t framerate) {
+ for (auto& encoder : encoders_) {
+ // TODO(emircan): |new_framerate| is used to calculate duration for encoder
+ // instances. We report the total frame rate to keep real time for now.
+ // Remove this after refactoring duration logic.
+ const int rv =
+ encoder->SetRateAllocation(bitrate, encoders_.size() * framerate);
+ if (rv)
+ return rv;
+ }
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int StereoEncoderAdapter::Release() {
+ for (auto& encoder : encoders_) {
+ const int rv = encoder->Release();
+ if (rv)
+ return rv;
+ }
+ encoders_.clear();
+ adapter_callbacks_.clear();
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+const char* StereoEncoderAdapter::ImplementationName() const {
+ return "StereoEncoderAdapter";
+}
+
+EncodedImageCallback::Result StereoEncoderAdapter::OnEncodedImage(
+ AlphaCodecStream stream_idx,
+ const EncodedImage& encodedImage,
+ const CodecSpecificInfo* codecSpecificInfo,
+ const RTPFragmentationHeader* fragmentation) {
+ // TODO(emircan): Fill |codec_specific_info| with stereo parameters.
+ encoded_complete_callback_->OnEncodedImage(encodedImage, codecSpecificInfo,
+ fragmentation);
+ return EncodedImageCallback::Result(EncodedImageCallback::Result::OK);
+}
+
+} // namespace webrtc
diff --git a/modules/video_coding/codecs/stereo/test/stereo_adapter_unittest.cc b/modules/video_coding/codecs/stereo/test/stereo_adapter_unittest.cc
new file mode 100644
index 0000000..a8913e7
--- /dev/null
+++ b/modules/video_coding/codecs/stereo/test/stereo_adapter_unittest.cc
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/test/mock_video_decoder_factory.h"
+#include "api/test/mock_video_encoder_factory.h"
+#include "common_video/libyuv/include/webrtc_libyuv.h"
+#include "modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h"
+#include "modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h"
+#include "modules/video_coding/codecs/test/video_codec_test.h"
+#include "modules/video_coding/codecs/vp9/include/vp9.h"
+
+using testing::_;
+using testing::Return;
+
+namespace webrtc {
+
+class TestStereoAdapter : public VideoCodecTest {
+ public:
+ TestStereoAdapter()
+ : decoder_factory_(new webrtc::MockVideoDecoderFactory),
+ encoder_factory_(new webrtc::MockVideoEncoderFactory) {}
+
+ protected:
+ VideoDecoder* CreateDecoder() override {
+ return new StereoDecoderAdapter(decoder_factory_.get());
+ }
+
+ VideoEncoder* CreateEncoder() override {
+ return new StereoEncoderAdapter(encoder_factory_.get());
+ }
+
+ VideoCodec codec_settings() override {
+ VideoCodec codec_settings;
+ codec_settings.codecType = webrtc::kVideoCodecVP9;
+ codec_settings.VP9()->numberOfTemporalLayers = 1;
+ codec_settings.VP9()->numberOfSpatialLayers = 1;
+ return codec_settings;
+ }
+
+ private:
+ void SetUp() override {
+ EXPECT_CALL(*decoder_factory_, Die());
+ VideoDecoder* decoder1 = VP9Decoder::Create();
+ VideoDecoder* decoder2 = VP9Decoder::Create();
+ EXPECT_CALL(*decoder_factory_, CreateVideoDecoderProxy(_))
+ .WillOnce(Return(decoder1))
+ .WillOnce(Return(decoder2));
+
+ EXPECT_CALL(*encoder_factory_, Die());
+ VideoEncoder* encoder1 = VP9Encoder::Create();
+ VideoEncoder* encoder2 = VP9Encoder::Create();
+ EXPECT_CALL(*encoder_factory_, CreateVideoEncoderProxy(_))
+ .WillOnce(Return(encoder1))
+ .WillOnce(Return(encoder2));
+
+ VideoCodecTest::SetUp();
+ }
+
+ const std::unique_ptr<webrtc::MockVideoDecoderFactory> decoder_factory_;
+ const std::unique_ptr<webrtc::MockVideoEncoderFactory> encoder_factory_;
+};
+
+// TODO(emircan): Currently VideoCodecTest tests do a complete setup
+// step that goes beyond constructing |decoder_|. Simplify these tests to do
+// less.
+TEST_F(TestStereoAdapter, ConstructAndDestructDecoder) {
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release());
+}
+
+TEST_F(TestStereoAdapter, ConstructAndDestructEncoder) {
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+}
+
+TEST_F(TestStereoAdapter, EncodeDecodeI420Frame) {
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->Encode(*input_frame_, nullptr, nullptr));
+ EncodedImage encoded_frame;
+ CodecSpecificInfo codec_specific_info;
+ ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ decoder_->Decode(encoded_frame, false, nullptr));
+ std::unique_ptr<VideoFrame> decoded_frame;
+ rtc::Optional<uint8_t> decoded_qp;
+ ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
+ ASSERT_TRUE(decoded_frame);
+ EXPECT_GT(I420PSNR(input_frame_.get(), decoded_frame.get()), 36);
+}
+
+} // namespace webrtc
