VideoStreamDecoderImpl implementation, part 3.
This CL implements the functions related to decoding.
Bug: webrtc:8909
Change-Id: Iefa3c1565a9b9ae93f14992b4a1cca141b7c5193
Reviewed-on: https://webrtc-review.googlesource.com/66403
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#22747}diff --git a/video/video_stream_decoder_impl.cc b/video/video_stream_decoder_impl.cc
index 3a9751e..7b4a1ba 100644
--- a/video/video_stream_decoder_impl.cc
+++ b/video/video_stream_decoder_impl.cc
@@ -11,6 +11,7 @@
#include "video/video_stream_decoder_impl.h"
#include "rtc_base/logging.h"
+#include "rtc_base/numerics/mod_ops.h"
#include "rtc_base/ptr_util.h"
namespace webrtc {
@@ -23,15 +24,24 @@
decoder_factory_(decoder_factory),
decoder_settings_(std::move(decoder_settings)),
bookkeeping_queue_("video_stream_decoder_bookkeeping_queue"),
+ decode_thread_(&DecodeLoop,
+ this,
+ "video_stream_decoder_decode_thread",
+ rtc::kHighestPriority),
jitter_estimator_(Clock::GetRealTimeClock()),
timing_(Clock::GetRealTimeClock()),
frame_buffer_(Clock::GetRealTimeClock(),
&jitter_estimator_,
&timing_,
- nullptr) {}
+ nullptr),
+ next_start_time_index_(0) {
+ decode_start_time_.fill({-1, 0});
+ decode_thread_.Start();
+}
VideoStreamDecoderImpl::~VideoStreamDecoderImpl() {
frame_buffer_.Stop();
+ decode_thread_.Stop();
}
void VideoStreamDecoderImpl::OnFrame(
@@ -111,4 +121,94 @@
return decoder_.get();
}
+// static
+void VideoStreamDecoderImpl::DecodeLoop(void* ptr) {
+ // TODO(philipel): Remove this and use rtc::Event::kForever when it's
+ // supported by the |frame_buffer_|.
+ static constexpr int kForever = 100000000;
+
+ int max_wait_time_ms = kForever;
+ bool keyframe_required = true;
+ auto* vs_decoder = static_cast<VideoStreamDecoderImpl*>(ptr);
+ while (true) {
+ DecodeResult decode_result =
+ vs_decoder->DecodeNextFrame(max_wait_time_ms, keyframe_required);
+
+ switch (decode_result) {
+ case kOk: {
+ max_wait_time_ms = kForever;
+ keyframe_required = false;
+ break;
+ }
+ case kDecodeFailure: {
+ max_wait_time_ms = 0;
+ keyframe_required = true;
+ break;
+ }
+ case kNoFrame: {
+ max_wait_time_ms = kForever;
+ // If we end up here it means that we got a decoding error and there is
+ // no keyframe available in the |frame_buffer_|.
+ vs_decoder->bookkeeping_queue_.PostTask([vs_decoder]() {
+ RTC_DCHECK_RUN_ON(&vs_decoder->bookkeeping_queue_);
+ vs_decoder->callbacks_->OnNonDecodableState();
+ });
+ break;
+ }
+ case kNoDecoder: {
+ max_wait_time_ms = kForever;
+ break;
+ }
+ case kShutdown: {
+ return;
+ }
+ }
+ }
+}
+
+VideoStreamDecoderImpl::DecodeResult VideoStreamDecoderImpl::DecodeNextFrame(
+ int max_wait_time_ms,
+ bool keyframe_required) {
+ std::unique_ptr<video_coding::EncodedFrame> frame;
+ video_coding::FrameBuffer::ReturnReason res =
+ frame_buffer_.NextFrame(max_wait_time_ms, &frame, keyframe_required);
+
+ if (res == video_coding::FrameBuffer::ReturnReason::kStopped)
+ return kShutdown;
+
+ if (frame) {
+ VideoDecoder* decoder = GetDecoder(frame->PayloadType());
+ if (!decoder) {
+ RTC_LOG(LS_WARNING) << "Failed to get decoder, dropping frame ("
+ << frame->id.picture_id << ":"
+ << frame->id.spatial_layer << ").";
+ return kNoDecoder;
+ }
+
+ int64_t decode_start_time_ms = rtc::TimeMillis();
+ uint32_t frame_timestamp = frame->timestamp;
+ bookkeeping_queue_.PostTask(
+ [this, decode_start_time_ms, frame_timestamp]() {
+ RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
+ // Saving decode start time this way wont work if we decode spatial
+ // layers sequentially.
+ decode_start_time_[next_start_time_index_] = {frame_timestamp,
+ decode_start_time_ms};
+ next_start_time_index_ =
+ Add<kDecodeTimeMemory>(next_start_time_index_, 1);
+ });
+
+ int32_t decode_result =
+ decoder->Decode(frame->EncodedImage(),
+ false, // missing_frame
+ nullptr, // rtp fragmentation header
+ nullptr, // codec specific info
+ frame->RenderTimeMs());
+
+ return decode_result == WEBRTC_VIDEO_CODEC_OK ? kOk : kDecodeFailure;
+ }
+
+ return kNoFrame;
+}
+
} // namespace webrtc
diff --git a/video/video_stream_decoder_impl.h b/video/video_stream_decoder_impl.h
index 5575176..42c1b94 100644
--- a/video/video_stream_decoder_impl.h
+++ b/video/video_stream_decoder_impl.h
@@ -20,6 +20,7 @@
#include "modules/video_coding/frame_buffer2.h"
#include "modules/video_coding/jitter_estimator.h"
#include "modules/video_coding/timing.h"
+#include "rtc_base/platform_thread.h"
#include "rtc_base/task_queue.h"
#include "rtc_base/thread_checker.h"
#include "system_wrappers/include/clock.h"
@@ -39,7 +40,17 @@
void OnFrame(std::unique_ptr<video_coding::EncodedFrame> frame) override;
private:
+ enum DecodeResult {
+ kOk,
+ kDecodeFailure,
+ kNoFrame,
+ kNoDecoder,
+ kShutdown,
+ };
+
VideoDecoder* GetDecoder(int payload_type);
+ static void DecodeLoop(void* ptr);
+ DecodeResult DecodeNextFrame(int max_wait_time_ms, bool keyframe_required);
// Implements DecodedImageCallback interface
int32_t Decoded(VideoFrame& decodedImage) override;
@@ -59,12 +70,21 @@
// - Synchronize with whatever thread that makes the Decoded callback.
rtc::TaskQueue bookkeeping_queue_;
+ rtc::PlatformThread decode_thread_;
VCMJitterEstimator jitter_estimator_;
VCMTiming timing_;
video_coding::FrameBuffer frame_buffer_;
video_coding::VideoLayerFrameId last_continuous_id_;
rtc::Optional<int> current_payload_type_;
std::unique_ptr<VideoDecoder> decoder_;
+
+ // Keep track of the |decode_start_time_| of the last |kDecodeTimeMemory|
+ // number of frames. The |decode_start_time_| array contain
+ // <frame timestamp> --> <decode start time> pairs.
+ static constexpr int kDecodeTimeMemory = 8;
+ std::array<std::pair<int64_t, int64_t>, kDecodeTimeMemory> decode_start_time_
+ RTC_GUARDED_BY(bookkeeping_queue_);
+ int next_start_time_index_ RTC_GUARDED_BY(bookkeeping_queue_);
};
} // namespace webrtc
