blob: fa1d2b921937277b72225a50f3244b242c92de3c [file] [log] [blame]
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h"
#include "api/video/encoded_image.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_frame_buffer.h"
#include "common_video/include/video_frame_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/video_coding/codecs/multiplex/include/augmented_video_frame_buffer.h"
#include "modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.h"
#include "rtc_base/keep_ref_until_done.h"
#include "rtc_base/logging.h"
namespace {
void KeepBufferRefs(rtc::scoped_refptr<webrtc::VideoFrameBuffer>,
rtc::scoped_refptr<webrtc::VideoFrameBuffer>) {}
} // anonymous namespace
namespace webrtc {
class MultiplexDecoderAdapter::AdapterDecodedImageCallback
: public webrtc::DecodedImageCallback {
public:
AdapterDecodedImageCallback(webrtc::MultiplexDecoderAdapter* adapter,
AlphaCodecStream stream_idx)
: adapter_(adapter), stream_idx_(stream_idx) {}
void Decoded(VideoFrame& decoded_image,
absl::optional<int32_t> decode_time_ms,
absl::optional<uint8_t> qp) override {
if (!adapter_)
return;
adapter_->Decoded(stream_idx_, &decoded_image, decode_time_ms, qp);
}
int32_t Decoded(VideoFrame& decoded_image) override {
RTC_NOTREACHED();
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t Decoded(VideoFrame& decoded_image, int64_t decode_time_ms) override {
RTC_NOTREACHED();
return WEBRTC_VIDEO_CODEC_OK;
}
private:
MultiplexDecoderAdapter* adapter_;
const AlphaCodecStream stream_idx_;
};
struct MultiplexDecoderAdapter::DecodedImageData {
explicit DecodedImageData(AlphaCodecStream stream_idx)
: stream_idx_(stream_idx),
decoded_image_(
VideoFrame::Builder()
.set_video_frame_buffer(
I420Buffer::Create(1 /* width */, 1 /* height */))
.set_timestamp_rtp(0)
.set_timestamp_us(0)
.set_rotation(kVideoRotation_0)
.build()) {
RTC_DCHECK_EQ(kAXXStream, stream_idx);
}
DecodedImageData(AlphaCodecStream stream_idx,
const VideoFrame& decoded_image,
const absl::optional<int32_t>& decode_time_ms,
const absl::optional<uint8_t>& qp)
: stream_idx_(stream_idx),
decoded_image_(decoded_image),
decode_time_ms_(decode_time_ms),
qp_(qp) {}
const AlphaCodecStream stream_idx_;
VideoFrame decoded_image_;
const absl::optional<int32_t> decode_time_ms_;
const absl::optional<uint8_t> qp_;
private:
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(DecodedImageData);
};
struct MultiplexDecoderAdapter::AugmentingData {
AugmentingData(std::unique_ptr<uint8_t[]> augmenting_data, uint16_t data_size)
: data_(std::move(augmenting_data)), size_(data_size) {}
std::unique_ptr<uint8_t[]> data_;
const uint16_t size_;
private:
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AugmentingData);
};
MultiplexDecoderAdapter::MultiplexDecoderAdapter(
VideoDecoderFactory* factory,
const SdpVideoFormat& associated_format,
bool supports_augmenting_data)
: factory_(factory),
associated_format_(associated_format),
supports_augmenting_data_(supports_augmenting_data) {}
MultiplexDecoderAdapter::~MultiplexDecoderAdapter() {
Release();
}
int32_t MultiplexDecoderAdapter::InitDecode(const VideoCodec* codec_settings,
int32_t number_of_cores) {
RTC_DCHECK_EQ(kVideoCodecMultiplex, codec_settings->codecType);
VideoCodec settings = *codec_settings;
settings.codecType = PayloadStringToCodecType(associated_format_.name);
for (size_t i = 0; i < kAlphaCodecStreams; ++i) {
std::unique_ptr<VideoDecoder> decoder =
factory_->CreateVideoDecoder(associated_format_);
const int32_t rv = decoder->InitDecode(&settings, number_of_cores);
if (rv)
return rv;
adapter_callbacks_.emplace_back(
new MultiplexDecoderAdapter::AdapterDecodedImageCallback(
this, static_cast<AlphaCodecStream>(i)));
decoder->RegisterDecodeCompleteCallback(adapter_callbacks_.back().get());
decoders_.emplace_back(std::move(decoder));
}
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t MultiplexDecoderAdapter::Decode(
const EncodedImage& input_image,
bool missing_frames,
int64_t render_time_ms) {
MultiplexImage image = MultiplexEncodedImagePacker::Unpack(input_image);
if (supports_augmenting_data_) {
RTC_DCHECK(decoded_augmenting_data_.find(input_image.Timestamp()) ==
decoded_augmenting_data_.end());
decoded_augmenting_data_.emplace(
std::piecewise_construct,
std::forward_as_tuple(input_image.Timestamp()),
std::forward_as_tuple(std::move(image.augmenting_data),
image.augmenting_data_size));
}
if (image.component_count == 1) {
RTC_DCHECK(decoded_data_.find(input_image.Timestamp()) ==
decoded_data_.end());
decoded_data_.emplace(std::piecewise_construct,
std::forward_as_tuple(input_image.Timestamp()),
std::forward_as_tuple(kAXXStream));
}
int32_t rv = 0;
for (size_t i = 0; i < image.image_components.size(); i++) {
rv = decoders_[image.image_components[i].component_index]->Decode(
image.image_components[i].encoded_image, missing_frames,
render_time_ms);
if (rv != WEBRTC_VIDEO_CODEC_OK)
return rv;
}
return rv;
}
int32_t MultiplexDecoderAdapter::RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) {
decoded_complete_callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t MultiplexDecoderAdapter::Release() {
for (auto& decoder : decoders_) {
const int32_t rv = decoder->Release();
if (rv)
return rv;
}
decoders_.clear();
adapter_callbacks_.clear();
return WEBRTC_VIDEO_CODEC_OK;
}
void MultiplexDecoderAdapter::Decoded(AlphaCodecStream stream_idx,
VideoFrame* decoded_image,
absl::optional<int32_t> decode_time_ms,
absl::optional<uint8_t> qp) {
const auto& other_decoded_data_it =
decoded_data_.find(decoded_image->timestamp());
const auto& augmenting_data_it =
decoded_augmenting_data_.find(decoded_image->timestamp());
const bool has_augmenting_data =
augmenting_data_it != decoded_augmenting_data_.end();
if (other_decoded_data_it != decoded_data_.end()) {
uint16_t augmenting_data_size =
has_augmenting_data ? augmenting_data_it->second.size_ : 0;
std::unique_ptr<uint8_t[]> augmenting_data =
has_augmenting_data ? std::move(augmenting_data_it->second.data_)
: nullptr;
auto& other_image_data = other_decoded_data_it->second;
if (stream_idx == kYUVStream) {
RTC_DCHECK_EQ(kAXXStream, other_image_data.stream_idx_);
MergeAlphaImages(decoded_image, decode_time_ms, qp,
&other_image_data.decoded_image_,
other_image_data.decode_time_ms_, other_image_data.qp_,
std::move(augmenting_data), augmenting_data_size);
} else {
RTC_DCHECK_EQ(kYUVStream, other_image_data.stream_idx_);
RTC_DCHECK_EQ(kAXXStream, stream_idx);
MergeAlphaImages(&other_image_data.decoded_image_,
other_image_data.decode_time_ms_, other_image_data.qp_,
decoded_image, decode_time_ms, qp,
std::move(augmenting_data), augmenting_data_size);
}
decoded_data_.erase(decoded_data_.begin(), other_decoded_data_it);
if (has_augmenting_data) {
decoded_augmenting_data_.erase(decoded_augmenting_data_.begin(),
augmenting_data_it);
}
return;
}
RTC_DCHECK(decoded_data_.find(decoded_image->timestamp()) ==
decoded_data_.end());
decoded_data_.emplace(
std::piecewise_construct,
std::forward_as_tuple(decoded_image->timestamp()),
std::forward_as_tuple(stream_idx, *decoded_image, decode_time_ms, qp));
}
void MultiplexDecoderAdapter::MergeAlphaImages(
VideoFrame* decoded_image,
const absl::optional<int32_t>& decode_time_ms,
const absl::optional<uint8_t>& qp,
VideoFrame* alpha_decoded_image,
const absl::optional<int32_t>& alpha_decode_time_ms,
const absl::optional<uint8_t>& alpha_qp,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_length) {
rtc::scoped_refptr<VideoFrameBuffer> merged_buffer;
if (!alpha_decoded_image->timestamp()) {
merged_buffer = decoded_image->video_frame_buffer();
} else {
rtc::scoped_refptr<webrtc::I420BufferInterface> yuv_buffer =
decoded_image->video_frame_buffer()->ToI420();
rtc::scoped_refptr<webrtc::I420BufferInterface> alpha_buffer =
alpha_decoded_image->video_frame_buffer()->ToI420();
RTC_DCHECK_EQ(yuv_buffer->width(), alpha_buffer->width());
RTC_DCHECK_EQ(yuv_buffer->height(), alpha_buffer->height());
merged_buffer = WrapI420ABuffer(
yuv_buffer->width(), yuv_buffer->height(), yuv_buffer->DataY(),
yuv_buffer->StrideY(), yuv_buffer->DataU(), yuv_buffer->StrideU(),
yuv_buffer->DataV(), yuv_buffer->StrideV(), alpha_buffer->DataY(),
alpha_buffer->StrideY(),
rtc::Bind(&KeepBufferRefs, yuv_buffer, alpha_buffer));
}
if (supports_augmenting_data_) {
merged_buffer = rtc::scoped_refptr<webrtc::AugmentedVideoFrameBuffer>(
new rtc::RefCountedObject<AugmentedVideoFrameBuffer>(
merged_buffer, std::move(augmenting_data), augmenting_data_length));
}
VideoFrame merged_image = VideoFrame::Builder()
.set_video_frame_buffer(merged_buffer)
.set_timestamp_rtp(decoded_image->timestamp())
.set_timestamp_us(0)
.set_rotation(decoded_image->rotation())
.set_id(decoded_image->id())
.set_packet_infos(decoded_image->packet_infos())
.build();
decoded_complete_callback_->Decoded(merged_image, decode_time_ms, qp);
}
} // namespace webrtc