video/video_stream_decoder_impl.cc - src/ - Git at Google

 /*
  *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */

 #include "video/video_stream_decoder_impl.h"

 #include "absl/memory/memory.h"
 #include "api/task_queue/queued_task.h"
 #include "rtc_base/logging.h"
 #include "rtc_base/numerics/mod_ops.h"
 #include "rtc_base/time_utils.h"

 namespace webrtc {

 VideoStreamDecoderImpl::VideoStreamDecoderImpl(
     VideoStreamDecoderInterface::Callbacks* callbacks,
     VideoDecoderFactory* decoder_factory,
     TaskQueueFactory* task_queue_factory,
     std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings)
     : callbacks_(callbacks),
       decoder_factory_(decoder_factory),
       decoder_settings_(std::move(decoder_settings)),
       bookkeeping_queue_(task_queue_factory->CreateTaskQueue(
           "video_stream_decoder_bookkeeping_queue",
           TaskQueueFactory::Priority::NORMAL)),
       decode_thread_(&DecodeLoop,
                      this,
                      "video_stream_decoder_decode_thread",
                      rtc::kHighestPriority),
       timing_(Clock::GetRealTimeClock()),
       frame_buffer_(Clock::GetRealTimeClock(),
                     &timing_,
                     nullptr),
       next_frame_timestamps_index_(0) {
   frame_timestamps_.fill({-1, -1, -1});
   decode_thread_.Start();
 }

 VideoStreamDecoderImpl::~VideoStreamDecoderImpl() {
   frame_buffer_.Stop();
   decode_thread_.Stop();
 }

 void VideoStreamDecoderImpl::OnFrame(
     std::unique_ptr<video_coding::EncodedFrame> frame) {
   if (!bookkeeping_queue_.IsCurrent()) {
     struct OnFrameTask : QueuedTask {
       OnFrameTask(std::unique_ptr<video_coding::EncodedFrame> frame,
                   VideoStreamDecoderImpl* video_stream_decoder)
           : frame_(std::move(frame)),
             video_stream_decoder_(video_stream_decoder) {}

       bool Run() override {
         video_stream_decoder_->OnFrame(std::move(frame_));
         return true;
       }

       std::unique_ptr<video_coding::EncodedFrame> frame_;
       VideoStreamDecoderImpl* video_stream_decoder_;
     };

     bookkeeping_queue_.PostTask(
         absl::make_unique<OnFrameTask>(std::move(frame), this));
     return;
   }

   RTC_DCHECK_RUN_ON(&bookkeeping_queue_);

   uint64_t continuous_pid = frame_buffer_.InsertFrame(std::move(frame));
   video_coding::VideoLayerFrameId continuous_id(continuous_pid, 0);
   if (last_continuous_id_ < continuous_id) {
     last_continuous_id_ = continuous_id;
     callbacks_->OnContinuousUntil(last_continuous_id_);
   }
 }

 VideoDecoder* VideoStreamDecoderImpl::GetDecoder(int payload_type) {
   if (current_payload_type_ == payload_type) {
     RTC_DCHECK(decoder_);
     return decoder_.get();
   }

   current_payload_type_.reset();
   decoder_.reset();

   auto decoder_settings_it = decoder_settings_.find(payload_type);
   if (decoder_settings_it == decoder_settings_.end()) {
     RTC_LOG(LS_WARNING) << "Payload type " << payload_type
                         << " not registered.";
     return nullptr;
   }

   const SdpVideoFormat& video_format = decoder_settings_it->second.first;
   std::unique_ptr<VideoDecoder> decoder =
       decoder_factory_->CreateVideoDecoder(video_format);
   if (!decoder) {
     RTC_LOG(LS_WARNING) << "Failed to create decoder for payload type "
                         << payload_type << ".";
     return nullptr;
   }

   int num_cores = decoder_settings_it->second.second;
   int32_t init_result = decoder->InitDecode(nullptr, num_cores);
   if (init_result != WEBRTC_VIDEO_CODEC_OK) {
     RTC_LOG(LS_WARNING) << "Failed to initialize decoder for payload type "
                         << payload_type << ".";
     return nullptr;
   }

   int32_t register_result = decoder->RegisterDecodeCompleteCallback(this);
   if (register_result != WEBRTC_VIDEO_CODEC_OK) {
     RTC_LOG(LS_WARNING) << "Failed to register decode callback.";
     return nullptr;
   }

   current_payload_type_.emplace(payload_type);
   decoder_ = std::move(decoder);
   return decoder_.get();
 }

 // static
 void VideoStreamDecoderImpl::DecodeLoop(void* ptr) {
   // TODO(philipel): Remove this and use rtc::Event::kForever when it's
   //                 supported by the |frame_buffer_|.
   static constexpr int kForever = 100000000;

   int max_wait_time_ms = kForever;
   bool keyframe_required = true;
   auto* vs_decoder = static_cast<VideoStreamDecoderImpl*>(ptr);
   while (true) {
     DecodeResult decode_result =
         vs_decoder->DecodeNextFrame(max_wait_time_ms, keyframe_required);

     switch (decode_result) {
       case kOk: {
         max_wait_time_ms = kForever;
         keyframe_required = false;
         break;
       }
       case kDecodeFailure: {
         max_wait_time_ms = 0;
         keyframe_required = true;
         break;
       }
       case kNoFrame: {
         max_wait_time_ms = kForever;
         // If we end up here it means that we got a decoding error and there is
         // no keyframe available in the |frame_buffer_|.
         vs_decoder->bookkeeping_queue_.PostTask([vs_decoder]() {
           RTC_DCHECK_RUN_ON(&vs_decoder->bookkeeping_queue_);
           vs_decoder->callbacks_->OnNonDecodableState();
         });
         break;
       }
       case kNoDecoder: {
         max_wait_time_ms = kForever;
         break;
       }
       case kShutdown: {
         return;
       }
     }
   }
 }

 VideoStreamDecoderImpl::DecodeResult VideoStreamDecoderImpl::DecodeNextFrame(
     int max_wait_time_ms,
     bool keyframe_required) {
   std::unique_ptr<video_coding::EncodedFrame> frame;
   video_coding::FrameBuffer::ReturnReason res =
       frame_buffer_.NextFrame(max_wait_time_ms, &frame, keyframe_required);

   if (res == video_coding::FrameBuffer::ReturnReason::kStopped)
     return kShutdown;

   if (frame) {
     VideoDecoder* decoder = GetDecoder(frame->PayloadType());
     if (!decoder) {
       RTC_LOG(LS_WARNING) << "Failed to get decoder, dropping frame ("
                           << frame->id.picture_id << ":"
                           << frame->id.spatial_layer << ").";
       return kNoDecoder;
     }

     int64_t decode_start_time_ms = rtc::TimeMillis();
     int64_t timestamp = frame->Timestamp();
     int64_t render_time_us = frame->RenderTimeMs() * 1000;
     bookkeeping_queue_.PostTask(
         [this, decode_start_time_ms, timestamp, render_time_us]() {
           RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
           // Saving decode start time this way wont work if we decode spatial
           // layers sequentially.
           FrameTimestamps* frame_timestamps =
               &frame_timestamps_[next_frame_timestamps_index_];
           frame_timestamps->timestamp = timestamp;
           frame_timestamps->decode_start_time_ms = decode_start_time_ms;
           frame_timestamps->render_time_us = render_time_us;

           next_frame_timestamps_index_ =
               Add<kFrameTimestampsMemory>(next_frame_timestamps_index_, 1);
         });

     int32_t decode_result = decoder->Decode(frame->EncodedImage(),
                                             false,    // missing_frame
                                             frame->RenderTimeMs());

     return decode_result == WEBRTC_VIDEO_CODEC_OK ? kOk : kDecodeFailure;
   }

   return kNoFrame;
 }

 VideoStreamDecoderImpl::FrameTimestamps*
 VideoStreamDecoderImpl::GetFrameTimestamps(int64_t timestamp) {
   RTC_DCHECK_RUN_ON(&bookkeeping_queue_);

   int start_time_index = next_frame_timestamps_index_;
   for (int i = 0; i < kFrameTimestampsMemory; ++i) {
     start_time_index = Subtract<kFrameTimestampsMemory>(start_time_index, 1);

     if (frame_timestamps_[start_time_index].timestamp == timestamp)
       return &frame_timestamps_[start_time_index];
   }

   return nullptr;
 }

 // VideoDecoder::DecodedImageCallback
 int32_t VideoStreamDecoderImpl::Decoded(VideoFrame& decoded_image) {
   Decoded(decoded_image, absl::nullopt, absl::nullopt);
   return WEBRTC_VIDEO_CODEC_OK;
 }

 // VideoDecoder::DecodedImageCallback
 int32_t VideoStreamDecoderImpl::Decoded(VideoFrame& decoded_image,
                                         int64_t decode_time_ms) {
   Decoded(decoded_image, decode_time_ms, absl::nullopt);
   return WEBRTC_VIDEO_CODEC_OK;
 }

 // VideoDecoder::DecodedImageCallback
 void VideoStreamDecoderImpl::Decoded(VideoFrame& decoded_image,
                                      absl::optional<int32_t> decode_time_ms,
                                      absl::optional<uint8_t> qp) {
   int64_t decode_stop_time_ms = rtc::TimeMillis();

   bookkeeping_queue_.PostTask([this, decode_stop_time_ms, decoded_image,
                                decode_time_ms, qp]() {
     RTC_DCHECK_RUN_ON(&bookkeeping_queue_);

     FrameTimestamps* frame_timestamps =
         GetFrameTimestamps(decoded_image.timestamp());
     if (!frame_timestamps) {
       RTC_LOG(LS_ERROR) << "No frame information found for frame with timestamp"
                         << decoded_image.timestamp();
       return;
     }

     absl::optional<int> casted_qp;
     if (qp)
       casted_qp.emplace(*qp);

     absl::optional<int> casted_decode_time_ms(decode_time_ms.value_or(
         decode_stop_time_ms - frame_timestamps->decode_start_time_ms));

     timing_.StopDecodeTimer(0, *casted_decode_time_ms, decode_stop_time_ms,
                             frame_timestamps->render_time_us / 1000);

     callbacks_->OnDecodedFrame(
         VideoFrame::Builder()
             .set_video_frame_buffer(decoded_image.video_frame_buffer())
             .set_rotation(decoded_image.rotation())
             .set_timestamp_us(frame_timestamps->render_time_us)
             .set_timestamp_rtp(decoded_image.timestamp())
             .set_id(decoded_image.id())
             .build(),
         casted_decode_time_ms, casted_qp);
   });
 }

 }  // namespace webrtc
	/*
	* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
	*
	* Use of this source code is governed by a BSD-style license
	* that can be found in the LICENSE file in the root of the source
	* tree. An additional intellectual property rights grant can be found
	* in the file PATENTS. All contributing project authors may
	* be found in the AUTHORS file in the root of the source tree.
	*/

	#include "video/video_stream_decoder_impl.h"

	#include "absl/memory/memory.h"
	#include "api/task_queue/queued_task.h"
	#include "rtc_base/logging.h"
	#include "rtc_base/numerics/mod_ops.h"
	#include "rtc_base/time_utils.h"

	namespace webrtc {

	VideoStreamDecoderImpl::VideoStreamDecoderImpl(
	VideoStreamDecoderInterface::Callbacks* callbacks,
	VideoDecoderFactory* decoder_factory,
	TaskQueueFactory* task_queue_factory,
	std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings)
	: callbacks_(callbacks),
	decoder_factory_(decoder_factory),
	decoder_settings_(std::move(decoder_settings)),
	bookkeeping_queue_(task_queue_factory->CreateTaskQueue(
	"video_stream_decoder_bookkeeping_queue",
	TaskQueueFactory::Priority::NORMAL)),
	decode_thread_(&DecodeLoop,
	this,
	"video_stream_decoder_decode_thread",
	rtc::kHighestPriority),
	timing_(Clock::GetRealTimeClock()),
	frame_buffer_(Clock::GetRealTimeClock(),
	&timing_,
	nullptr),
	next_frame_timestamps_index_(0) {
	frame_timestamps_.fill({-1, -1, -1});
	decode_thread_.Start();
	}

	VideoStreamDecoderImpl::~VideoStreamDecoderImpl() {
	frame_buffer_.Stop();
	decode_thread_.Stop();
	}

	void VideoStreamDecoderImpl::OnFrame(
	std::unique_ptr<video_coding::EncodedFrame> frame) {
	if (!bookkeeping_queue_.IsCurrent()) {
	struct OnFrameTask : QueuedTask {
	OnFrameTask(std::unique_ptr<video_coding::EncodedFrame> frame,
	VideoStreamDecoderImpl* video_stream_decoder)
	: frame_(std::move(frame)),
	video_stream_decoder_(video_stream_decoder) {}

	bool Run() override {
	video_stream_decoder_->OnFrame(std::move(frame_));
	return true;
	}

	std::unique_ptr<video_coding::EncodedFrame> frame_;
	VideoStreamDecoderImpl* video_stream_decoder_;
	};

	bookkeeping_queue_.PostTask(
	absl::make_unique<OnFrameTask>(std::move(frame), this));
	return;
	}

	RTC_DCHECK_RUN_ON(&bookkeeping_queue_);

	uint64_t continuous_pid = frame_buffer_.InsertFrame(std::move(frame));
	video_coding::VideoLayerFrameId continuous_id(continuous_pid, 0);
	if (last_continuous_id_ < continuous_id) {
	last_continuous_id_ = continuous_id;
	callbacks_->OnContinuousUntil(last_continuous_id_);
	}
	}

	VideoDecoder* VideoStreamDecoderImpl::GetDecoder(int payload_type) {
	if (current_payload_type_ == payload_type) {
	RTC_DCHECK(decoder_);
	return decoder_.get();
	}

	current_payload_type_.reset();
	decoder_.reset();

	auto decoder_settings_it = decoder_settings_.find(payload_type);
	if (decoder_settings_it == decoder_settings_.end()) {
	RTC_LOG(LS_WARNING) << "Payload type " << payload_type
	<< " not registered.";
	return nullptr;
	}

	const SdpVideoFormat& video_format = decoder_settings_it->second.first;
	std::unique_ptr<VideoDecoder> decoder =
	decoder_factory_->CreateVideoDecoder(video_format);
	if (!decoder) {
	RTC_LOG(LS_WARNING) << "Failed to create decoder for payload type "
	<< payload_type << ".";
	return nullptr;
	}

	int num_cores = decoder_settings_it->second.second;
	int32_t init_result = decoder->InitDecode(nullptr, num_cores);
	if (init_result != WEBRTC_VIDEO_CODEC_OK) {
	RTC_LOG(LS_WARNING) << "Failed to initialize decoder for payload type "
	<< payload_type << ".";
	return nullptr;
	}

	int32_t register_result = decoder->RegisterDecodeCompleteCallback(this);
	if (register_result != WEBRTC_VIDEO_CODEC_OK) {
	RTC_LOG(LS_WARNING) << "Failed to register decode callback.";
	return nullptr;
	}

	current_payload_type_.emplace(payload_type);
	decoder_ = std::move(decoder);
	return decoder_.get();
	}

	// static
	void VideoStreamDecoderImpl::DecodeLoop(void* ptr) {
	// TODO(philipel): Remove this and use rtc::Event::kForever when it's
	// supported by the \|frame_buffer_\|.
	static constexpr int kForever = 100000000;

	int max_wait_time_ms = kForever;
	bool keyframe_required = true;
	auto* vs_decoder = static_cast<VideoStreamDecoderImpl*>(ptr);
	while (true) {
	DecodeResult decode_result =
	vs_decoder->DecodeNextFrame(max_wait_time_ms, keyframe_required);

	switch (decode_result) {
	case kOk: {
	max_wait_time_ms = kForever;
	keyframe_required = false;
	break;
	}
	case kDecodeFailure: {
	max_wait_time_ms = 0;
	keyframe_required = true;
	break;
	}
	case kNoFrame: {
	max_wait_time_ms = kForever;
	// If we end up here it means that we got a decoding error and there is
	// no keyframe available in the \|frame_buffer_\|.
	vs_decoder->bookkeeping_queue_.PostTask([vs_decoder]() {
	RTC_DCHECK_RUN_ON(&vs_decoder->bookkeeping_queue_);
	vs_decoder->callbacks_->OnNonDecodableState();
	});
	break;
	}
	case kNoDecoder: {
	max_wait_time_ms = kForever;
	break;
	}
	case kShutdown: {
	return;
	}
	}
	}
	}

	VideoStreamDecoderImpl::DecodeResult VideoStreamDecoderImpl::DecodeNextFrame(
	int max_wait_time_ms,
	bool keyframe_required) {
	std::unique_ptr<video_coding::EncodedFrame> frame;
	video_coding::FrameBuffer::ReturnReason res =
	frame_buffer_.NextFrame(max_wait_time_ms, &frame, keyframe_required);

	if (res == video_coding::FrameBuffer::ReturnReason::kStopped)
	return kShutdown;

	if (frame) {
	VideoDecoder* decoder = GetDecoder(frame->PayloadType());
	if (!decoder) {
	RTC_LOG(LS_WARNING) << "Failed to get decoder, dropping frame ("
	<< frame->id.picture_id << ":"
	<< frame->id.spatial_layer << ").";
	return kNoDecoder;
	}

	int64_t decode_start_time_ms = rtc::TimeMillis();
	int64_t timestamp = frame->Timestamp();
	int64_t render_time_us = frame->RenderTimeMs() * 1000;
	bookkeeping_queue_.PostTask(
	[this, decode_start_time_ms, timestamp, render_time_us]() {
	RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
	// Saving decode start time this way wont work if we decode spatial
	// layers sequentially.
	FrameTimestamps* frame_timestamps =
	&frame_timestamps_[next_frame_timestamps_index_];
	frame_timestamps->timestamp = timestamp;
	frame_timestamps->decode_start_time_ms = decode_start_time_ms;
	frame_timestamps->render_time_us = render_time_us;

	next_frame_timestamps_index_ =
	Add<kFrameTimestampsMemory>(next_frame_timestamps_index_, 1);
	});

	int32_t decode_result = decoder->Decode(frame->EncodedImage(),
	false, // missing_frame
	frame->RenderTimeMs());

	return decode_result == WEBRTC_VIDEO_CODEC_OK ? kOk : kDecodeFailure;
	}

	return kNoFrame;
	}

	VideoStreamDecoderImpl::FrameTimestamps*
	VideoStreamDecoderImpl::GetFrameTimestamps(int64_t timestamp) {
	RTC_DCHECK_RUN_ON(&bookkeeping_queue_);

	int start_time_index = next_frame_timestamps_index_;
	for (int i = 0; i < kFrameTimestampsMemory; ++i) {
	start_time_index = Subtract<kFrameTimestampsMemory>(start_time_index, 1);

	if (frame_timestamps_[start_time_index].timestamp == timestamp)
	return &frame_timestamps_[start_time_index];
	}

	return nullptr;
	}

	// VideoDecoder::DecodedImageCallback
	int32_t VideoStreamDecoderImpl::Decoded(VideoFrame& decoded_image) {
	Decoded(decoded_image, absl::nullopt, absl::nullopt);
	return WEBRTC_VIDEO_CODEC_OK;
	}

	// VideoDecoder::DecodedImageCallback
	int32_t VideoStreamDecoderImpl::Decoded(VideoFrame& decoded_image,
	int64_t decode_time_ms) {
	Decoded(decoded_image, decode_time_ms, absl::nullopt);
	return WEBRTC_VIDEO_CODEC_OK;
	}

	// VideoDecoder::DecodedImageCallback
	void VideoStreamDecoderImpl::Decoded(VideoFrame& decoded_image,
	absl::optional<int32_t> decode_time_ms,
	absl::optional<uint8_t> qp) {
	int64_t decode_stop_time_ms = rtc::TimeMillis();

	bookkeeping_queue_.PostTask([this, decode_stop_time_ms, decoded_image,
	decode_time_ms, qp]() {
	RTC_DCHECK_RUN_ON(&bookkeeping_queue_);

	FrameTimestamps* frame_timestamps =
	GetFrameTimestamps(decoded_image.timestamp());
	if (!frame_timestamps) {
	RTC_LOG(LS_ERROR) << "No frame information found for frame with timestamp"
	<< decoded_image.timestamp();
	return;
	}

	absl::optional<int> casted_qp;
	if (qp)
	casted_qp.emplace(*qp);

	absl::optional<int> casted_decode_time_ms(decode_time_ms.value_or(
	decode_stop_time_ms - frame_timestamps->decode_start_time_ms));

	timing_.StopDecodeTimer(0, *casted_decode_time_ms, decode_stop_time_ms,
	frame_timestamps->render_time_us / 1000);

	callbacks_->OnDecodedFrame(
	VideoFrame::Builder()
	.set_video_frame_buffer(decoded_image.video_frame_buffer())
	.set_rotation(decoded_image.rotation())
	.set_timestamp_us(frame_timestamps->render_time_us)
	.set_timestamp_rtp(decoded_image.timestamp())
	.set_id(decoded_image.id())
	.build(),
	casted_decode_time_ms, casted_qp);
	});
	}

	} // namespace webrtc