philipel | 2fee4d6 | 2018-03-21 15:52:13 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
| 11 | #ifndef VIDEO_VIDEO_STREAM_DECODER_IMPL_H_ |
| 12 | #define VIDEO_VIDEO_STREAM_DECODER_IMPL_H_ |
| 13 | |
philipel | 2fee4d6 | 2018-03-21 15:52:13 | [diff] [blame] | 14 | #include <map> |
| 15 | #include <memory> |
| 16 | #include <utility> |
| 17 | |
Danil Chapovalov | b9b146c | 2018-06-15 10:28:07 | [diff] [blame] | 18 | #include "absl/types/optional.h" |
Artem Titov | d15a575 | 2021-02-10 13:31:24 | [diff] [blame] | 19 | #include "api/sequence_checker.h" |
philipel | 2fee4d6 | 2018-03-21 15:52:13 | [diff] [blame] | 20 | #include "api/video/video_stream_decoder.h" |
philipel | 9718711 | 2018-03-23 09:43:21 | [diff] [blame] | 21 | #include "modules/video_coding/frame_buffer2.h" |
philipel | 9718711 | 2018-03-23 09:43:21 | [diff] [blame] | 22 | #include "modules/video_coding/timing.h" |
philipel | 844876d | 2018-04-05 09:02:54 | [diff] [blame] | 23 | #include "rtc_base/platform_thread.h" |
Markus Handell | 265931e | 2020-07-10 10:23:48 | [diff] [blame] | 24 | #include "rtc_base/synchronization/mutex.h" |
philipel | 9718711 | 2018-03-23 09:43:21 | [diff] [blame] | 25 | #include "rtc_base/task_queue.h" |
philipel | 9718711 | 2018-03-23 09:43:21 | [diff] [blame] | 26 | #include "system_wrappers/include/clock.h" |
philipel | 2fee4d6 | 2018-03-21 15:52:13 | [diff] [blame] | 27 | |
| 28 | namespace webrtc { |
| 29 | |
philipel | 539f9b3 | 2020-01-09 15:12:25 | [diff] [blame] | 30 | class VideoStreamDecoderImpl : public VideoStreamDecoderInterface { |
philipel | 2fee4d6 | 2018-03-21 15:52:13 | [diff] [blame] | 31 | public: |
| 32 | VideoStreamDecoderImpl( |
Danil Chapovalov | b703db9 | 2019-04-08 14:59:28 | [diff] [blame] | 33 | VideoStreamDecoderInterface::Callbacks* callbacks, |
philipel | 2fee4d6 | 2018-03-21 15:52:13 | [diff] [blame] | 34 | VideoDecoderFactory* decoder_factory, |
Danil Chapovalov | b703db9 | 2019-04-08 14:59:28 | [diff] [blame] | 35 | TaskQueueFactory* task_queue_factory, |
philipel | 2fee4d6 | 2018-03-21 15:52:13 | [diff] [blame] | 36 | std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings); |
| 37 | |
| 38 | ~VideoStreamDecoderImpl() override; |
| 39 | |
philipel | ca18809 | 2021-03-23 11:00:49 | [diff] [blame] | 40 | void OnFrame(std::unique_ptr<EncodedFrame> frame) override; |
philipel | 2fee4d6 | 2018-03-21 15:52:13 | [diff] [blame] | 41 | |
philipel | 781653c | 2019-06-04 15:10:37 | [diff] [blame] | 42 | void SetMinPlayoutDelay(TimeDelta min_delay) override; |
| 43 | void SetMaxPlayoutDelay(TimeDelta max_delay) override; |
| 44 | |
philipel | 2fee4d6 | 2018-03-21 15:52:13 | [diff] [blame] | 45 | private: |
philipel | 539f9b3 | 2020-01-09 15:12:25 | [diff] [blame] | 46 | class DecodeCallbacks : public DecodedImageCallback { |
| 47 | public: |
| 48 | explicit DecodeCallbacks(VideoStreamDecoderImpl* video_stream_decoder_impl); |
| 49 | int32_t Decoded(VideoFrame& decodedImage) override; |
| 50 | int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override; |
| 51 | void Decoded(VideoFrame& decodedImage, |
| 52 | absl::optional<int32_t> decode_time_ms, |
| 53 | absl::optional<uint8_t> qp) override; |
| 54 | |
| 55 | private: |
| 56 | VideoStreamDecoderImpl* const video_stream_decoder_impl_; |
| 57 | }; |
| 58 | |
philipel | 844876d | 2018-04-05 09:02:54 | [diff] [blame] | 59 | enum DecodeResult { |
| 60 | kOk, |
philipel | 539f9b3 | 2020-01-09 15:12:25 | [diff] [blame] | 61 | kOkRequestKeyframe, |
philipel | 844876d | 2018-04-05 09:02:54 | [diff] [blame] | 62 | kDecodeFailure, |
philipel | 844876d | 2018-04-05 09:02:54 | [diff] [blame] | 63 | }; |
| 64 | |
philipel | 3dc4780 | 2020-09-10 13:30:02 | [diff] [blame] | 65 | struct FrameInfo { |
| 66 | int64_t timestamp = -1; |
philipel | 6847f9b | 2018-04-20 13:05:37 | [diff] [blame] | 67 | int64_t decode_start_time_ms; |
| 68 | int64_t render_time_us; |
philipel | 3dc4780 | 2020-09-10 13:30:02 | [diff] [blame] | 69 | VideoContentType content_type; |
philipel | 6847f9b | 2018-04-20 13:05:37 | [diff] [blame] | 70 | }; |
| 71 | |
philipel | ca18809 | 2021-03-23 11:00:49 | [diff] [blame] | 72 | void SaveFrameInfo(const EncodedFrame& frame) RTC_RUN_ON(bookkeeping_queue_); |
philipel | 3dc4780 | 2020-09-10 13:30:02 | [diff] [blame] | 73 | FrameInfo* GetFrameInfo(int64_t timestamp) RTC_RUN_ON(bookkeeping_queue_); |
philipel | 539f9b3 | 2020-01-09 15:12:25 | [diff] [blame] | 74 | void StartNextDecode() RTC_RUN_ON(bookkeeping_queue_); |
philipel | ca18809 | 2021-03-23 11:00:49 | [diff] [blame] | 75 | void OnNextFrameCallback(std::unique_ptr<EncodedFrame> frame, |
philipel | 539f9b3 | 2020-01-09 15:12:25 | [diff] [blame] | 76 | video_coding::FrameBuffer::ReturnReason res) |
| 77 | RTC_RUN_ON(bookkeeping_queue_); |
| 78 | void OnDecodedFrameCallback(VideoFrame& decodedImage, // NOLINT |
| 79 | absl::optional<int32_t> decode_time_ms, |
| 80 | absl::optional<uint8_t> qp); |
philipel | 79aab3f | 2018-03-26 12:31:23 | [diff] [blame] | 81 | |
philipel | 539f9b3 | 2020-01-09 15:12:25 | [diff] [blame] | 82 | VideoDecoder* GetDecoder(int payload_type) RTC_RUN_ON(decode_queue_); |
| 83 | VideoStreamDecoderImpl::DecodeResult DecodeFrame( |
philipel | ca18809 | 2021-03-23 11:00:49 | [diff] [blame] | 84 | std::unique_ptr<EncodedFrame> frame) RTC_RUN_ON(decode_queue_); |
philipel | 6847f9b | 2018-04-20 13:05:37 | [diff] [blame] | 85 | |
philipel | 9718711 | 2018-03-23 09:43:21 | [diff] [blame] | 86 | VCMTiming timing_; |
philipel | 539f9b3 | 2020-01-09 15:12:25 | [diff] [blame] | 87 | DecodeCallbacks decode_callbacks_; |
philipel | 844876d | 2018-04-05 09:02:54 | [diff] [blame] | 88 | |
philipel | 6847f9b | 2018-04-20 13:05:37 | [diff] [blame] | 89 | // Some decoders are pipelined so it is not sufficient to save frame info |
| 90 | // for the last frame only. |
philipel | 3dc4780 | 2020-09-10 13:30:02 | [diff] [blame] | 91 | static constexpr int kFrameInfoMemory = 8; |
| 92 | std::array<FrameInfo, kFrameInfoMemory> frame_info_ |
philipel | 844876d | 2018-04-05 09:02:54 | [diff] [blame] | 93 | RTC_GUARDED_BY(bookkeeping_queue_); |
philipel | 3dc4780 | 2020-09-10 13:30:02 | [diff] [blame] | 94 | int next_frame_info_index_ RTC_GUARDED_BY(bookkeeping_queue_); |
philipel | 539f9b3 | 2020-01-09 15:12:25 | [diff] [blame] | 95 | VideoStreamDecoderInterface::Callbacks* const callbacks_ |
| 96 | RTC_PT_GUARDED_BY(bookkeeping_queue_); |
philipel | 9aa9b8d | 2021-02-15 12:31:29 | [diff] [blame] | 97 | int64_t last_continuous_frame_id_ RTC_GUARDED_BY(bookkeeping_queue_) = -1; |
philipel | 539f9b3 | 2020-01-09 15:12:25 | [diff] [blame] | 98 | bool keyframe_required_ RTC_GUARDED_BY(bookkeeping_queue_); |
| 99 | |
| 100 | absl::optional<int> current_payload_type_ RTC_GUARDED_BY(decode_queue_); |
| 101 | VideoDecoderFactory* const decoder_factory_ RTC_PT_GUARDED_BY(decode_queue_); |
| 102 | std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings_ |
| 103 | RTC_GUARDED_BY(decode_queue_); |
| 104 | |
Artem Titov | ab30d72 | 2021-07-27 14:22:11 | [diff] [blame] | 105 | // The `bookkeeping_queue_` use the `frame_buffer_` and also posts tasks to |
| 106 | // the `decode_queue_`. The `decode_queue_` in turn use the `decoder_` to |
| 107 | // decode frames. When the `decoder_` is done it will post back to the |
| 108 | // `bookkeeping_queue_` with the decoded frame. During shutdown we start by |
| 109 | // isolating the `bookkeeping_queue_` from the `decode_queue_`, so now it's |
| 110 | // safe for the `decode_queue_` to be destructed. After that the `decoder_` |
| 111 | // can be destructed, and then the `bookkeeping_queue_`. Finally the |
| 112 | // `frame_buffer_` can be destructed. |
Markus Handell | 265931e | 2020-07-10 10:23:48 | [diff] [blame] | 113 | Mutex shut_down_mutex_; |
| 114 | bool shut_down_ RTC_GUARDED_BY(shut_down_mutex_); |
philipel | 539f9b3 | 2020-01-09 15:12:25 | [diff] [blame] | 115 | video_coding::FrameBuffer frame_buffer_ RTC_GUARDED_BY(bookkeeping_queue_); |
| 116 | rtc::TaskQueue bookkeeping_queue_; |
| 117 | std::unique_ptr<VideoDecoder> decoder_ RTC_GUARDED_BY(decode_queue_); |
| 118 | rtc::TaskQueue decode_queue_; |
philipel | 2fee4d6 | 2018-03-21 15:52:13 | [diff] [blame] | 119 | }; |
| 120 | |
| 121 | } // namespace webrtc |
| 122 | |
| 123 | #endif // VIDEO_VIDEO_STREAM_DECODER_IMPL_H_ |