niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 1 | /* |
pwestin@webrtc.org | 52fd98d | 2012-02-13 09:03:53 | [diff] [blame] | 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 11 | #ifndef MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_ |
| 12 | #define MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_ |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 13 | |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 14 | #include "modules/video_coding/include/video_coding.h" |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 15 | |
kwiberg | 3f55dea | 2016-02-29 13:51:59 | [diff] [blame] | 16 | #include <memory> |
perkj | 376b192 | 2016-05-02 18:35:24 | [diff] [blame] | 17 | #include <string> |
stefan@webrtc.org | c530043 | 2012-10-08 07:06:53 | [diff] [blame] | 18 | #include <vector> |
| 19 | |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 20 | #include "common_video/include/frame_callback.h" |
Niels Möller | f906378 | 2018-02-20 15:09:48 | [diff] [blame] | 21 | #include "modules/video_coding/decoder_database.h" |
| 22 | #include "modules/video_coding/encoder_database.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 23 | #include "modules/video_coding/frame_buffer.h" |
| 24 | #include "modules/video_coding/generic_decoder.h" |
| 25 | #include "modules/video_coding/generic_encoder.h" |
| 26 | #include "modules/video_coding/jitter_buffer.h" |
| 27 | #include "modules/video_coding/media_optimization.h" |
| 28 | #include "modules/video_coding/qp_parser.h" |
| 29 | #include "modules/video_coding/receiver.h" |
| 30 | #include "modules/video_coding/timing.h" |
| 31 | #include "rtc_base/onetimeevent.h" |
| 32 | #include "rtc_base/sequenced_task_checker.h" |
| 33 | #include "rtc_base/thread_annotations.h" |
| 34 | #include "rtc_base/thread_checker.h" |
| 35 | #include "system_wrappers/include/clock.h" |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 36 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 37 | namespace webrtc { |
sprang@webrtc.org | 4070935 | 2013-11-26 11:41:59 | [diff] [blame] | 38 | |
Erik Språng | 08127a9 | 2016-11-16 15:41:30 | [diff] [blame] | 39 | class VideoBitrateAllocator; |
sprang | 1a646ee | 2016-12-01 14:34:11 | [diff] [blame] | 40 | class VideoBitrateAllocationObserver; |
Erik Språng | 08127a9 | 2016-11-16 15:41:30 | [diff] [blame] | 41 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 42 | namespace vcm { |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 43 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 44 | class VCMProcessTimer { |
| 45 | public: |
sprang | 40217c3 | 2016-11-21 13:41:52 | [diff] [blame] | 46 | static const int64_t kDefaultProcessIntervalMs = 1000; |
| 47 | |
pkasting@chromium.org | 0b1534c | 2014-12-15 22:09:40 | [diff] [blame] | 48 | VCMProcessTimer(int64_t periodMs, Clock* clock) |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 49 | : _clock(clock), |
| 50 | _periodMs(periodMs), |
| 51 | _latestMs(_clock->TimeInMilliseconds()) {} |
pkasting@chromium.org | 0b1534c | 2014-12-15 22:09:40 | [diff] [blame] | 52 | int64_t Period() const; |
| 53 | int64_t TimeUntilProcess() const; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 54 | void Processed(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 55 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 56 | private: |
| 57 | Clock* _clock; |
pkasting@chromium.org | 0b1534c | 2014-12-15 22:09:40 | [diff] [blame] | 58 | int64_t _periodMs; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 59 | int64_t _latestMs; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 60 | }; |
| 61 | |
Niels Möller | a056599 | 2017-10-24 09:37:08 | [diff] [blame] | 62 | class VideoSender { |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 63 | public: |
| 64 | typedef VideoCodingModule::SenderNackMode SenderNackMode; |
| 65 | |
Yves Gerey | 665174f | 2018-06-19 13:03:05 | [diff] [blame] | 66 | VideoSender(Clock* clock, EncodedImageCallback* post_encode_callback); |
andresp@webrtc.org | 1df9dc3 | 2014-01-09 08:01:57 | [diff] [blame] | 67 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 68 | ~VideoSender(); |
| 69 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 70 | // Register the send codec to be used. |
tommi@webrtc.org | e07710c | 2015-02-19 17:43:25 | [diff] [blame] | 71 | // This method must be called on the construction thread. |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 72 | int32_t RegisterSendCodec(const VideoCodec* sendCodec, |
| 73 | uint32_t numberOfCores, |
| 74 | uint32_t maxPayloadSize); |
tommi@webrtc.org | e07710c | 2015-02-19 17:43:25 | [diff] [blame] | 75 | |
Peter Boström | 795dbe4 | 2015-11-27 13:09:07 | [diff] [blame] | 76 | void RegisterExternalEncoder(VideoEncoder* externalEncoder, |
Peter Boström | 795dbe4 | 2015-11-27 13:09:07 | [diff] [blame] | 77 | bool internalSource); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 78 | |
sprang | 1a646ee | 2016-12-01 14:34:11 | [diff] [blame] | 79 | // Update the channel parameters based on new rates and rtt. This will also |
| 80 | // cause an immediate call to VideoEncoder::SetRateAllocation(). |
| 81 | int32_t SetChannelParameters( |
| 82 | uint32_t target_bitrate_bps, |
| 83 | uint8_t loss_rate, |
| 84 | int64_t rtt, |
| 85 | VideoBitrateAllocator* bitrate_allocator, |
| 86 | VideoBitrateAllocationObserver* bitrate_updated_callback); |
| 87 | |
| 88 | // Updates the channel parameters with a new bitrate allocation, but using the |
| 89 | // current targit_bitrate, loss rate and rtt. That is, the distribution or |
| 90 | // caps may be updated to a change to a new VideoCodec or allocation mode. |
| 91 | // The new parameters will be stored as pending EncoderParameters, and the |
| 92 | // encoder will only be updated on the next frame. |
Niels Möller | 96d7f76 | 2018-01-30 10:27:16 | [diff] [blame] | 93 | void UpdateChannelParameters( |
sprang | 1a646ee | 2016-12-01 14:34:11 | [diff] [blame] | 94 | VideoBitrateAllocator* bitrate_allocator, |
| 95 | VideoBitrateAllocationObserver* bitrate_updated_callback); |
Erik Språng | 08127a9 | 2016-11-16 15:41:30 | [diff] [blame] | 96 | |
Miguel Casas-Sanchez | 4765070 | 2015-05-30 00:21:40 | [diff] [blame] | 97 | int32_t AddVideoFrame(const VideoFrame& videoFrame, |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 98 | const CodecSpecificInfo* codecSpecificInfo); |
| 99 | |
perkj | 600246e | 2016-05-04 18:26:51 | [diff] [blame] | 100 | int32_t IntraFrameRequest(size_t stream_index); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 101 | int32_t EnableFrameDropper(bool enable); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 102 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 103 | private: |
Erik Språng | 08127a9 | 2016-11-16 15:41:30 | [diff] [blame] | 104 | EncoderParameters UpdateEncoderParameters( |
| 105 | const EncoderParameters& params, |
| 106 | VideoBitrateAllocator* bitrate_allocator, |
| 107 | uint32_t target_bitrate_bps); |
perkj | 57c21f9 | 2016-06-17 14:27:16 | [diff] [blame] | 108 | void SetEncoderParameters(EncoderParameters params, bool has_internal_source) |
danilchap | 56359be | 2017-09-07 14:53:45 | [diff] [blame] | 109 | RTC_EXCLUSIVE_LOCKS_REQUIRED(encoder_crit_); |
Peter Boström | dcb8998 | 2015-09-15 12:43:47 | [diff] [blame] | 110 | |
pbos | 5ad935c | 2016-01-25 11:52:44 | [diff] [blame] | 111 | rtc::CriticalSection encoder_crit_; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 112 | VCMGenericEncoder* _encoder; |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 | [diff] [blame] | 113 | media_optimization::MediaOptimization _mediaOpt; |
danilchap | 56359be | 2017-09-07 14:53:45 | [diff] [blame] | 114 | VCMEncodedFrameCallback _encodedFrameCallback RTC_GUARDED_BY(encoder_crit_); |
kthelgason | 876222f | 2016-11-29 09:44:11 | [diff] [blame] | 115 | EncodedImageCallback* const post_encode_callback_; |
Niels Möller | f906378 | 2018-02-20 15:09:48 | [diff] [blame] | 116 | VCMEncoderDataBase _codecDataBase RTC_GUARDED_BY(encoder_crit_); |
danilchap | 56359be | 2017-09-07 14:53:45 | [diff] [blame] | 117 | bool frame_dropper_enabled_ RTC_GUARDED_BY(encoder_crit_); |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 | [diff] [blame] | 118 | |
tommi@webrtc.org | e07710c | 2015-02-19 17:43:25 | [diff] [blame] | 119 | // Must be accessed on the construction thread of VideoSender. |
| 120 | VideoCodec current_codec_; |
perkj | 4e417b2 | 2016-07-15 06:35:55 | [diff] [blame] | 121 | rtc::SequencedTaskChecker sequenced_checker_; |
Erik Språng | 66a641a | 2015-06-11 12:20:07 | [diff] [blame] | 122 | |
Peter Boström | 233bfd2 | 2016-01-18 19:23:40 | [diff] [blame] | 123 | rtc::CriticalSection params_crit_; |
danilchap | 56359be | 2017-09-07 14:53:45 | [diff] [blame] | 124 | EncoderParameters encoder_params_ RTC_GUARDED_BY(params_crit_); |
| 125 | bool encoder_has_internal_source_ RTC_GUARDED_BY(params_crit_); |
| 126 | std::vector<FrameType> next_frame_types_ RTC_GUARDED_BY(params_crit_); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 127 | }; |
| 128 | |
Peter Boström | 0b25072 | 2016-04-22 16:23:15 | [diff] [blame] | 129 | class VideoReceiver : public Module { |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 130 | public: |
philipel | 83f831a | 2016-03-12 11:30:23 | [diff] [blame] | 131 | VideoReceiver(Clock* clock, |
| 132 | EventFactory* event_factory, |
sprang | 3911c26 | 2016-04-15 08:24:14 | [diff] [blame] | 133 | EncodedImageCallback* pre_decode_image_callback, |
philipel | 721d402 | 2016-12-15 15:10:57 | [diff] [blame] | 134 | VCMTiming* timing, |
philipel | 83f831a | 2016-03-12 11:30:23 | [diff] [blame] | 135 | NackSender* nack_sender = nullptr, |
| 136 | KeyFrameRequestSender* keyframe_request_sender = nullptr); |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 137 | ~VideoReceiver() override; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 138 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 139 | int32_t RegisterReceiveCodec(const VideoCodec* receiveCodec, |
| 140 | int32_t numberOfCores, |
| 141 | bool requireKeyFrame); |
| 142 | |
Peter Boström | 795dbe4 | 2015-11-27 13:09:07 | [diff] [blame] | 143 | void RegisterExternalDecoder(VideoDecoder* externalDecoder, |
perkj | 796cfaf | 2015-12-10 17:27:38 | [diff] [blame] | 144 | uint8_t payloadType); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 145 | int32_t RegisterReceiveCallback(VCMReceiveCallback* receiveCallback); |
| 146 | int32_t RegisterReceiveStatisticsCallback( |
| 147 | VCMReceiveStatisticsCallback* receiveStats); |
| 148 | int32_t RegisterFrameTypeCallback(VCMFrameTypeCallback* frameTypeCallback); |
| 149 | int32_t RegisterPacketRequestCallback(VCMPacketRequestCallback* callback); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 150 | |
| 151 | int32_t Decode(uint16_t maxWaitTimeMs); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 152 | |
philipel | fd5a20f | 2016-11-15 08:57:57 | [diff] [blame] | 153 | int32_t Decode(const webrtc::VCMEncodedFrame* frame); |
| 154 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 155 | int32_t IncomingPacket(const uint8_t* incomingPayload, |
pkasting@chromium.org | 4591fbd | 2014-11-20 22:28:14 | [diff] [blame] | 156 | size_t payloadLength, |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 157 | const WebRtcRTPHeader& rtpInfo); |
| 158 | int32_t SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs); |
| 159 | int32_t SetRenderDelay(uint32_t timeMS); |
| 160 | int32_t Delay() const; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 161 | |
tommi | a5c18d7 | 2017-03-20 17:43:23 | [diff] [blame] | 162 | // DEPRECATED. |
| 163 | int SetReceiverRobustnessMode( |
| 164 | VideoCodingModule::ReceiverRobustness robustnessMode, |
| 165 | VCMDecodeErrorMode errorMode); |
| 166 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 167 | void SetNackSettings(size_t max_nack_list_size, |
| 168 | int max_packet_age_to_nack, |
| 169 | int max_incomplete_time_ms); |
| 170 | |
| 171 | void SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 172 | |
pkasting@chromium.org | 16825b1 | 2015-01-12 21:51:21 | [diff] [blame] | 173 | int32_t SetReceiveChannelParameters(int64_t rtt); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 174 | int32_t SetVideoProtection(VCMVideoProtection videoProtection, bool enable); |
| 175 | |
Peter Boström | 0b25072 | 2016-04-22 16:23:15 | [diff] [blame] | 176 | int64_t TimeUntilNextProcess() override; |
| 177 | void Process() override; |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 178 | void ProcessThreadAttached(ProcessThread* process_thread) override; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 179 | |
pbos@webrtc.org | 4dd40d6 | 2015-02-17 13:22:43 | [diff] [blame] | 180 | void TriggerDecoderShutdown(); |
sprang@webrtc.org | 4070935 | 2013-11-26 11:41:59 | [diff] [blame] | 181 | |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 182 | // Notification methods that are used to check our internal state and validate |
| 183 | // threading assumptions. These are called by VideoReceiveStream. |
| 184 | // See |IsDecoderThreadRunning()| for more details. |
| 185 | void DecoderThreadStarting(); |
| 186 | void DecoderThreadStopped(); |
| 187 | |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 188 | protected: |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 189 | int32_t Decode(const webrtc::VCMEncodedFrame& frame); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 190 | int32_t RequestKeyFrame(); |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 191 | |
| 192 | private: |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 193 | // Used for DCHECKing thread correctness. |
| 194 | // In build where DCHECKs are enabled, will return false before |
| 195 | // DecoderThreadStarting is called, then true until DecoderThreadStopped |
| 196 | // is called. |
| 197 | // In builds where DCHECKs aren't enabled, it will return true. |
| 198 | bool IsDecoderThreadRunning(); |
| 199 | |
| 200 | rtc::ThreadChecker construction_thread_checker_; |
| 201 | rtc::ThreadChecker decoder_thread_checker_; |
| 202 | rtc::ThreadChecker module_thread_checker_; |
pbos@webrtc.org | 20c1f56 | 2014-07-04 10:58:12 | [diff] [blame] | 203 | Clock* const clock_; |
sprang | 3911c26 | 2016-04-15 08:24:14 | [diff] [blame] | 204 | rtc::CriticalSection process_crit_; |
philipel | 721d402 | 2016-12-15 15:10:57 | [diff] [blame] | 205 | VCMTiming* _timing; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 206 | VCMReceiver _receiver; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 207 | VCMDecodedFrameCallback _decodedFrameCallback; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 208 | |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 209 | // These callbacks are set on the construction thread before being attached |
| 210 | // to the module thread or decoding started, so a lock is not required. |
| 211 | VCMFrameTypeCallback* _frameTypeCallback; |
| 212 | VCMReceiveStatisticsCallback* _receiveStatsCallback; |
| 213 | VCMPacketRequestCallback* _packetRequestCallback; |
| 214 | |
| 215 | // Used on both the module and decoder thread. |
danilchap | 56359be | 2017-09-07 14:53:45 | [diff] [blame] | 216 | bool _scheduleKeyRequest RTC_GUARDED_BY(process_crit_); |
| 217 | bool drop_frames_until_keyframe_ RTC_GUARDED_BY(process_crit_); |
sprang | 3911c26 | 2016-04-15 08:24:14 | [diff] [blame] | 218 | |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 219 | // Modified on the construction thread while not attached to the process |
| 220 | // thread. Once attached to the process thread, its value is only read |
| 221 | // so a lock is not required. |
| 222 | size_t max_nack_list_size_; |
Peter Boström | ed3277b | 2016-02-02 14:40:04 | [diff] [blame] | 223 | |
Tommi | fbf3bce | 2018-02-21 14:56:05 | [diff] [blame] | 224 | // Callbacks are set before the decoder thread starts. |
| 225 | // Once the decoder thread has been started, usage of |_codecDataBase| moves |
| 226 | // over to the decoder thread. |
| 227 | VCMDecoderDataBase _codecDataBase; |
| 228 | EncodedImageCallback* const pre_decode_image_callback_; |
| 229 | |
| 230 | VCMProcessTimer _receiveStatsTimer RTC_GUARDED_BY(module_thread_checker_); |
| 231 | VCMProcessTimer _retransmissionTimer RTC_GUARDED_BY(module_thread_checker_); |
| 232 | VCMProcessTimer _keyRequestTimer RTC_GUARDED_BY(module_thread_checker_); |
| 233 | QpParser qp_parser_ RTC_GUARDED_BY(decoder_thread_checker_); |
| 234 | ThreadUnsafeOneTimeEvent first_frame_received_ |
| 235 | RTC_GUARDED_BY(decoder_thread_checker_); |
| 236 | // Modified on the construction thread. Can be read without a lock and assumed |
| 237 | // to be non-null on the module and decoder threads. |
| 238 | ProcessThread* process_thread_ = nullptr; |
| 239 | bool is_attached_to_process_thread_ |
| 240 | RTC_GUARDED_BY(construction_thread_checker_) = false; |
| 241 | #if RTC_DCHECK_IS_ON |
| 242 | bool decoder_thread_is_running_ = false; |
| 243 | #endif |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 | [diff] [blame] | 244 | }; |
andresp@webrtc.org | f7eb75b | 2013-09-14 00:25:28 | [diff] [blame] | 245 | |
| 246 | } // namespace vcm |
pbos@webrtc.org | d900e8b | 2013-07-03 15:12:26 | [diff] [blame] | 247 | } // namespace webrtc |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 248 | #endif // MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_ |