niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 1 | /* |
pwestin@webrtc.org | 52fd98d | 2012-02-13 09:03:53 | [diff] [blame] | 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 11 | #ifndef MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_ |
| 12 | #define MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_ |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 13 | |
kwiberg | 3f55dea | 2016-02-29 13:51:59 | [diff] [blame] | 14 | #include <memory> |
perkj | 376b192 | 2016-05-02 18:35:24 | [diff] [blame] | 15 | #include <string> |
stefan@webrtc.org | c530043 | 2012-10-08 07:06:53 | [diff] [blame] | 16 | #include <vector> |
| 17 | |
Erik Språng | eee3920 | 2018-11-15 16:52:43 | [diff] [blame] | 18 | #include "absl/types/optional.h" |
Artem Titov | d15a575 | 2021-02-10 13:31:24 | [diff] [blame] | 19 | #include "api/sequence_checker.h" |
Niels Möller | f906378 | 2018-02-20 15:09:48 | [diff] [blame] | 20 | #include "modules/video_coding/decoder_database.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 21 | #include "modules/video_coding/frame_buffer.h" |
| 22 | #include "modules/video_coding/generic_decoder.h" |
Jonas Olsson | a4d8737 | 2019-07-05 17:08:33 | [diff] [blame] | 23 | #include "modules/video_coding/include/video_coding.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 24 | #include "modules/video_coding/jitter_buffer.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 25 | #include "modules/video_coding/receiver.h" |
| 26 | #include "modules/video_coding/timing.h" |
Steve Anton | 10542f2 | 2019-01-11 17:11:00 | [diff] [blame] | 27 | #include "rtc_base/one_time_event.h" |
Markus Handell | 6deec38 | 2020-07-07 10:17:12 | [diff] [blame] | 28 | #include "rtc_base/synchronization/mutex.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 29 | #include "rtc_base/thread_annotations.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 30 | #include "system_wrappers/include/clock.h" |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 31 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 32 | namespace webrtc { |
sprang@webrtc.org | 4070935 | 2013-11-26 11:41:59 | [diff] [blame] | 33 | |
Erik Språng | 08127a9 | 2016-11-16 15:41:30 | [diff] [blame] | 34 | class VideoBitrateAllocator; |
sprang | 1a646ee | 2016-12-01 14:34:11 | [diff] [blame] | 35 | class VideoBitrateAllocationObserver; |
Erik Språng | 08127a9 | 2016-11-16 15:41:30 | [diff] [blame] | 36 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 37 | namespace vcm { |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 38 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 39 | class VCMProcessTimer { |
| 40 | public: |
sprang | 40217c3 | 2016-11-21 13:41:52 | [diff] [blame] | 41 | static const int64_t kDefaultProcessIntervalMs = 1000; |
| 42 | |
pkasting@chromium.org | 0b1534c | 2014-12-15 22:09:40 | [diff] [blame] | 43 | VCMProcessTimer(int64_t periodMs, Clock* clock) |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 44 | : _clock(clock), |
| 45 | _periodMs(periodMs), |
| 46 | _latestMs(_clock->TimeInMilliseconds()) {} |
pkasting@chromium.org | 0b1534c | 2014-12-15 22:09:40 | [diff] [blame] | 47 | int64_t Period() const; |
| 48 | int64_t TimeUntilProcess() const; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 49 | void Processed(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 50 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 51 | private: |
| 52 | Clock* _clock; |
pkasting@chromium.org | 0b1534c | 2014-12-15 22:09:40 | [diff] [blame] | 53 | int64_t _periodMs; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 54 | int64_t _latestMs; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 55 | }; |
| 56 | |
Peter Boström | 0b25072 | 2016-04-22 16:23:15 | [diff] [blame] | 57 | class VideoReceiver : public Module { |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 58 | public: |
Niels Möller | db64d99 | 2019-03-29 13:30:53 | [diff] [blame] | 59 | VideoReceiver(Clock* clock, VCMTiming* timing); |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 60 | ~VideoReceiver() override; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 61 | |
Niels Möller | 582102c | 2020-08-07 14:19:56 | [diff] [blame] | 62 | int32_t RegisterReceiveCodec(uint8_t payload_type, |
| 63 | const VideoCodec* receiveCodec, |
Niels Möller | 18c83d3 | 2020-08-07 12:14:49 | [diff] [blame] | 64 | int32_t numberOfCores); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 65 | |
Peter Boström | 795dbe4 | 2015-11-27 13:09:07 | [diff] [blame] | 66 | void RegisterExternalDecoder(VideoDecoder* externalDecoder, |
perkj | 796cfaf | 2015-12-10 17:27:38 | [diff] [blame] | 67 | uint8_t payloadType); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 68 | int32_t RegisterReceiveCallback(VCMReceiveCallback* receiveCallback); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 69 | int32_t RegisterFrameTypeCallback(VCMFrameTypeCallback* frameTypeCallback); |
| 70 | int32_t RegisterPacketRequestCallback(VCMPacketRequestCallback* callback); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 71 | |
| 72 | int32_t Decode(uint16_t maxWaitTimeMs); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 73 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 74 | int32_t IncomingPacket(const uint8_t* incomingPayload, |
pkasting@chromium.org | 4591fbd | 2014-11-20 22:28:14 | [diff] [blame] | 75 | size_t payloadLength, |
Niels Möller | be7a0ec | 2019-04-25 08:02:52 | [diff] [blame] | 76 | const RTPHeader& rtp_header, |
| 77 | const RTPVideoHeader& video_header); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 78 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 79 | void SetNackSettings(size_t max_nack_list_size, |
| 80 | int max_packet_age_to_nack, |
| 81 | int max_incomplete_time_ms); |
| 82 | |
Peter Boström | 0b25072 | 2016-04-22 16:23:15 | [diff] [blame] | 83 | int64_t TimeUntilNextProcess() override; |
| 84 | void Process() override; |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 85 | void ProcessThreadAttached(ProcessThread* process_thread) override; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 86 | |
| 87 | protected: |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 88 | int32_t Decode(const webrtc::VCMEncodedFrame& frame); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 89 | int32_t RequestKeyFrame(); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 90 | |
| 91 | private: |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 92 | // Used for DCHECKing thread correctness. |
| 93 | // In build where DCHECKs are enabled, will return false before |
| 94 | // DecoderThreadStarting is called, then true until DecoderThreadStopped |
| 95 | // is called. |
| 96 | // In builds where DCHECKs aren't enabled, it will return true. |
| 97 | bool IsDecoderThreadRunning(); |
| 98 | |
Artem Titov | c8421c4 | 2021-02-02 09:57:19 | [diff] [blame] | 99 | SequenceChecker construction_thread_checker_; |
| 100 | SequenceChecker decoder_thread_checker_; |
| 101 | SequenceChecker module_thread_checker_; |
pbos@webrtc.org | 20c1f56 | 2014-07-04 10:58:12 | [diff] [blame] | 102 | Clock* const clock_; |
Markus Handell | 6deec38 | 2020-07-07 10:17:12 | [diff] [blame] | 103 | Mutex process_mutex_; |
philipel | 721d402 | 2016-12-15 15:10:57 | [diff] [blame] | 104 | VCMTiming* _timing; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 105 | VCMReceiver _receiver; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 106 | VCMDecodedFrameCallback _decodedFrameCallback; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 107 | |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 108 | // These callbacks are set on the construction thread before being attached |
| 109 | // to the module thread or decoding started, so a lock is not required. |
| 110 | VCMFrameTypeCallback* _frameTypeCallback; |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 111 | VCMPacketRequestCallback* _packetRequestCallback; |
| 112 | |
| 113 | // Used on both the module and decoder thread. |
Markus Handell | 6deec38 | 2020-07-07 10:17:12 | [diff] [blame] | 114 | bool _scheduleKeyRequest RTC_GUARDED_BY(process_mutex_); |
| 115 | bool drop_frames_until_keyframe_ RTC_GUARDED_BY(process_mutex_); |
sprang | 3911c26 | 2016-04-15 08:24:14 | [diff] [blame] | 116 | |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 117 | // Modified on the construction thread while not attached to the process |
| 118 | // thread. Once attached to the process thread, its value is only read |
| 119 | // so a lock is not required. |
| 120 | size_t max_nack_list_size_; |
Peter Boström | ed3277b | 2016-02-02 14:40:04 | [diff] [blame] | 121 | |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 122 | // Callbacks are set before the decoder thread starts. |
| 123 | // Once the decoder thread has been started, usage of |_codecDataBase| moves |
| 124 | // over to the decoder thread. |
| 125 | VCMDecoderDataBase _codecDataBase; |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 126 | |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 127 | VCMProcessTimer _retransmissionTimer RTC_GUARDED_BY(module_thread_checker_); |
| 128 | VCMProcessTimer _keyRequestTimer RTC_GUARDED_BY(module_thread_checker_); |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 129 | ThreadUnsafeOneTimeEvent first_frame_received_ |
| 130 | RTC_GUARDED_BY(decoder_thread_checker_); |
| 131 | // Modified on the construction thread. Can be read without a lock and assumed |
| 132 | // to be non-null on the module and decoder threads. |
| 133 | ProcessThread* process_thread_ = nullptr; |
| 134 | bool is_attached_to_process_thread_ |
| 135 | RTC_GUARDED_BY(construction_thread_checker_) = false; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 136 | }; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 137 | |
| 138 | } // namespace vcm |
pbos@webrtc.org | d900e8b | 2013-07-03 15:12:26 | [diff] [blame] | 139 | } // namespace webrtc |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 140 | #endif // MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_ |