blob: 2e28b0fbe9938ea1c8b57fff2487c0e4b35580e6 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:251/*
pwestin@webrtc.org52fd98d2012-02-13 09:03:532 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:253 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 04:47:3111#ifndef MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_
12#define MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_
niklase@google.com470e71d2011-07-07 08:21:2513
Mirko Bonadei92ea95e2017-09-15 04:47:3114#include "modules/video_coding/include/video_coding.h"
niklase@google.com470e71d2011-07-07 08:21:2515
kwiberg3f55dea2016-02-29 13:51:5916#include <memory>
perkj376b1922016-05-02 18:35:2417#include <string>
stefan@webrtc.orgc5300432012-10-08 07:06:5318#include <vector>
19
Mirko Bonadei92ea95e2017-09-15 04:47:3120#include "common_video/include/frame_callback.h"
Niels Möllerf9063782018-02-20 15:09:4821#include "modules/video_coding/decoder_database.h"
22#include "modules/video_coding/encoder_database.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3123#include "modules/video_coding/frame_buffer.h"
24#include "modules/video_coding/generic_decoder.h"
25#include "modules/video_coding/generic_encoder.h"
26#include "modules/video_coding/jitter_buffer.h"
27#include "modules/video_coding/media_optimization.h"
28#include "modules/video_coding/qp_parser.h"
29#include "modules/video_coding/receiver.h"
30#include "modules/video_coding/timing.h"
31#include "rtc_base/onetimeevent.h"
32#include "rtc_base/sequenced_task_checker.h"
33#include "rtc_base/thread_annotations.h"
34#include "rtc_base/thread_checker.h"
35#include "system_wrappers/include/clock.h"
niklase@google.com470e71d2011-07-07 08:21:2536
andresp@webrtc.orgf7eb75b2013-09-14 00:25:2837namespace webrtc {
sprang@webrtc.org40709352013-11-26 11:41:5938
Erik Språng08127a92016-11-16 15:41:3039class VideoBitrateAllocator;
sprang1a646ee2016-12-01 14:34:1140class VideoBitrateAllocationObserver;
Erik Språng08127a92016-11-16 15:41:3041
andresp@webrtc.orgf7eb75b2013-09-14 00:25:2842namespace vcm {
niklase@google.com470e71d2011-07-07 08:21:2543
andresp@webrtc.orgf7eb75b2013-09-14 00:25:2844class VCMProcessTimer {
45 public:
sprang40217c32016-11-21 13:41:5246 static const int64_t kDefaultProcessIntervalMs = 1000;
47
pkasting@chromium.org0b1534c2014-12-15 22:09:4048 VCMProcessTimer(int64_t periodMs, Clock* clock)
andresp@webrtc.orgf7eb75b2013-09-14 00:25:2849 : _clock(clock),
50 _periodMs(periodMs),
51 _latestMs(_clock->TimeInMilliseconds()) {}
pkasting@chromium.org0b1534c2014-12-15 22:09:4052 int64_t Period() const;
53 int64_t TimeUntilProcess() const;
andresp@webrtc.orgf7eb75b2013-09-14 00:25:2854 void Processed();
niklase@google.com470e71d2011-07-07 08:21:2555
andresp@webrtc.orgf7eb75b2013-09-14 00:25:2856 private:
57 Clock* _clock;
pkasting@chromium.org0b1534c2014-12-15 22:09:4058 int64_t _periodMs;
andresp@webrtc.orgf7eb75b2013-09-14 00:25:2859 int64_t _latestMs;
niklase@google.com470e71d2011-07-07 08:21:2560};
61
Niels Möllera0565992017-10-24 09:37:0862class VideoSender {
andresp@webrtc.orgf7eb75b2013-09-14 00:25:2863 public:
64 typedef VideoCodingModule::SenderNackMode SenderNackMode;
65
Yves Gerey665174f2018-06-19 13:03:0566 VideoSender(Clock* clock, EncodedImageCallback* post_encode_callback);
andresp@webrtc.org1df9dc32014-01-09 08:01:5767
andresp@webrtc.orgf7eb75b2013-09-14 00:25:2868 ~VideoSender();
69
andresp@webrtc.orgf7eb75b2013-09-14 00:25:2870 // Register the send codec to be used.
tommi@webrtc.orge07710c2015-02-19 17:43:2571 // This method must be called on the construction thread.
andresp@webrtc.orgf7eb75b2013-09-14 00:25:2872 int32_t RegisterSendCodec(const VideoCodec* sendCodec,
73 uint32_t numberOfCores,
74 uint32_t maxPayloadSize);
tommi@webrtc.orge07710c2015-02-19 17:43:2575
Peter Boström795dbe42015-11-27 13:09:0776 void RegisterExternalEncoder(VideoEncoder* externalEncoder,
Peter Boström795dbe42015-11-27 13:09:0777 bool internalSource);
andresp@webrtc.orgf7eb75b2013-09-14 00:25:2878
sprang1a646ee2016-12-01 14:34:1179 // Update the channel parameters based on new rates and rtt. This will also
80 // cause an immediate call to VideoEncoder::SetRateAllocation().
81 int32_t SetChannelParameters(
82 uint32_t target_bitrate_bps,
83 uint8_t loss_rate,
84 int64_t rtt,
85 VideoBitrateAllocator* bitrate_allocator,
86 VideoBitrateAllocationObserver* bitrate_updated_callback);
87
88 // Updates the channel parameters with a new bitrate allocation, but using the
89 // current targit_bitrate, loss rate and rtt. That is, the distribution or
90 // caps may be updated to a change to a new VideoCodec or allocation mode.
91 // The new parameters will be stored as pending EncoderParameters, and the
92 // encoder will only be updated on the next frame.
Niels Möller96d7f762018-01-30 10:27:1693 void UpdateChannelParameters(
sprang1a646ee2016-12-01 14:34:1194 VideoBitrateAllocator* bitrate_allocator,
95 VideoBitrateAllocationObserver* bitrate_updated_callback);
Erik Språng08127a92016-11-16 15:41:3096
Miguel Casas-Sanchez47650702015-05-30 00:21:4097 int32_t AddVideoFrame(const VideoFrame& videoFrame,
andresp@webrtc.orgf7eb75b2013-09-14 00:25:2898 const CodecSpecificInfo* codecSpecificInfo);
99
perkj600246e2016-05-04 18:26:51100 int32_t IntraFrameRequest(size_t stream_index);
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28101 int32_t EnableFrameDropper(bool enable);
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28102
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28103 private:
Erik Språng08127a92016-11-16 15:41:30104 EncoderParameters UpdateEncoderParameters(
105 const EncoderParameters& params,
106 VideoBitrateAllocator* bitrate_allocator,
107 uint32_t target_bitrate_bps);
perkj57c21f92016-06-17 14:27:16108 void SetEncoderParameters(EncoderParameters params, bool has_internal_source)
danilchap56359be2017-09-07 14:53:45109 RTC_EXCLUSIVE_LOCKS_REQUIRED(encoder_crit_);
Peter Boströmdcb89982015-09-15 12:43:47110
pbos5ad935c2016-01-25 11:52:44111 rtc::CriticalSection encoder_crit_;
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28112 VCMGenericEncoder* _encoder;
henrik.lundin@webrtc.orgbec11ef2013-09-23 19:54:25113 media_optimization::MediaOptimization _mediaOpt;
danilchap56359be2017-09-07 14:53:45114 VCMEncodedFrameCallback _encodedFrameCallback RTC_GUARDED_BY(encoder_crit_);
kthelgason876222f2016-11-29 09:44:11115 EncodedImageCallback* const post_encode_callback_;
Niels Möllerf9063782018-02-20 15:09:48116 VCMEncoderDataBase _codecDataBase RTC_GUARDED_BY(encoder_crit_);
danilchap56359be2017-09-07 14:53:45117 bool frame_dropper_enabled_ RTC_GUARDED_BY(encoder_crit_);
andresp@webrtc.orge682aa52013-12-19 10:59:48118
tommi@webrtc.orge07710c2015-02-19 17:43:25119 // Must be accessed on the construction thread of VideoSender.
120 VideoCodec current_codec_;
perkj4e417b22016-07-15 06:35:55121 rtc::SequencedTaskChecker sequenced_checker_;
Erik Språng66a641a2015-06-11 12:20:07122
Peter Boström233bfd22016-01-18 19:23:40123 rtc::CriticalSection params_crit_;
danilchap56359be2017-09-07 14:53:45124 EncoderParameters encoder_params_ RTC_GUARDED_BY(params_crit_);
125 bool encoder_has_internal_source_ RTC_GUARDED_BY(params_crit_);
126 std::vector<FrameType> next_frame_types_ RTC_GUARDED_BY(params_crit_);
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28127};
128
Peter Boström0b250722016-04-22 16:23:15129class VideoReceiver : public Module {
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28130 public:
philipel83f831a2016-03-12 11:30:23131 VideoReceiver(Clock* clock,
132 EventFactory* event_factory,
sprang3911c262016-04-15 08:24:14133 EncodedImageCallback* pre_decode_image_callback,
philipel721d4022016-12-15 15:10:57134 VCMTiming* timing,
philipel83f831a2016-03-12 11:30:23135 NackSender* nack_sender = nullptr,
136 KeyFrameRequestSender* keyframe_request_sender = nullptr);
Tommifbf3bce2018-02-21 14:56:05137 ~VideoReceiver() override;
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28138
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28139 int32_t RegisterReceiveCodec(const VideoCodec* receiveCodec,
140 int32_t numberOfCores,
141 bool requireKeyFrame);
142
Peter Boström795dbe42015-11-27 13:09:07143 void RegisterExternalDecoder(VideoDecoder* externalDecoder,
perkj796cfaf2015-12-10 17:27:38144 uint8_t payloadType);
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28145 int32_t RegisterReceiveCallback(VCMReceiveCallback* receiveCallback);
146 int32_t RegisterReceiveStatisticsCallback(
147 VCMReceiveStatisticsCallback* receiveStats);
148 int32_t RegisterFrameTypeCallback(VCMFrameTypeCallback* frameTypeCallback);
149 int32_t RegisterPacketRequestCallback(VCMPacketRequestCallback* callback);
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28150
151 int32_t Decode(uint16_t maxWaitTimeMs);
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28152
philipelfd5a20f2016-11-15 08:57:57153 int32_t Decode(const webrtc::VCMEncodedFrame* frame);
154
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28155 int32_t IncomingPacket(const uint8_t* incomingPayload,
pkasting@chromium.org4591fbd2014-11-20 22:28:14156 size_t payloadLength,
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28157 const WebRtcRTPHeader& rtpInfo);
158 int32_t SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs);
159 int32_t SetRenderDelay(uint32_t timeMS);
160 int32_t Delay() const;
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28161
tommia5c18d72017-03-20 17:43:23162 // DEPRECATED.
163 int SetReceiverRobustnessMode(
164 VideoCodingModule::ReceiverRobustness robustnessMode,
165 VCMDecodeErrorMode errorMode);
166
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28167 void SetNackSettings(size_t max_nack_list_size,
168 int max_packet_age_to_nack,
169 int max_incomplete_time_ms);
170
171 void SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode);
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28172
pkasting@chromium.org16825b12015-01-12 21:51:21173 int32_t SetReceiveChannelParameters(int64_t rtt);
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28174 int32_t SetVideoProtection(VCMVideoProtection videoProtection, bool enable);
175
Peter Boström0b250722016-04-22 16:23:15176 int64_t TimeUntilNextProcess() override;
177 void Process() override;
Tommifbf3bce2018-02-21 14:56:05178 void ProcessThreadAttached(ProcessThread* process_thread) override;
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28179
pbos@webrtc.org4dd40d62015-02-17 13:22:43180 void TriggerDecoderShutdown();
sprang@webrtc.org40709352013-11-26 11:41:59181
Tommifbf3bce2018-02-21 14:56:05182 // Notification methods that are used to check our internal state and validate
183 // threading assumptions. These are called by VideoReceiveStream.
184 // See |IsDecoderThreadRunning()| for more details.
185 void DecoderThreadStarting();
186 void DecoderThreadStopped();
187
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28188 protected:
Tommifbf3bce2018-02-21 14:56:05189 int32_t Decode(const webrtc::VCMEncodedFrame& frame);
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28190 int32_t RequestKeyFrame();
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28191
192 private:
Tommifbf3bce2018-02-21 14:56:05193 // Used for DCHECKing thread correctness.
194 // In build where DCHECKs are enabled, will return false before
195 // DecoderThreadStarting is called, then true until DecoderThreadStopped
196 // is called.
197 // In builds where DCHECKs aren't enabled, it will return true.
198 bool IsDecoderThreadRunning();
199
200 rtc::ThreadChecker construction_thread_checker_;
201 rtc::ThreadChecker decoder_thread_checker_;
202 rtc::ThreadChecker module_thread_checker_;
pbos@webrtc.org20c1f562014-07-04 10:58:12203 Clock* const clock_;
sprang3911c262016-04-15 08:24:14204 rtc::CriticalSection process_crit_;
philipel721d4022016-12-15 15:10:57205 VCMTiming* _timing;
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28206 VCMReceiver _receiver;
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28207 VCMDecodedFrameCallback _decodedFrameCallback;
niklase@google.com470e71d2011-07-07 08:21:25208
Tommifbf3bce2018-02-21 14:56:05209 // These callbacks are set on the construction thread before being attached
210 // to the module thread or decoding started, so a lock is not required.
211 VCMFrameTypeCallback* _frameTypeCallback;
212 VCMReceiveStatisticsCallback* _receiveStatsCallback;
213 VCMPacketRequestCallback* _packetRequestCallback;
214
215 // Used on both the module and decoder thread.
danilchap56359be2017-09-07 14:53:45216 bool _scheduleKeyRequest RTC_GUARDED_BY(process_crit_);
217 bool drop_frames_until_keyframe_ RTC_GUARDED_BY(process_crit_);
sprang3911c262016-04-15 08:24:14218
Tommifbf3bce2018-02-21 14:56:05219 // Modified on the construction thread while not attached to the process
220 // thread. Once attached to the process thread, its value is only read
221 // so a lock is not required.
222 size_t max_nack_list_size_;
Peter Boströmed3277b2016-02-02 14:40:04223
Tommifbf3bce2018-02-21 14:56:05224 // Callbacks are set before the decoder thread starts.
225 // Once the decoder thread has been started, usage of |_codecDataBase| moves
226 // over to the decoder thread.
227 VCMDecoderDataBase _codecDataBase;
228 EncodedImageCallback* const pre_decode_image_callback_;
229
230 VCMProcessTimer _receiveStatsTimer RTC_GUARDED_BY(module_thread_checker_);
231 VCMProcessTimer _retransmissionTimer RTC_GUARDED_BY(module_thread_checker_);
232 VCMProcessTimer _keyRequestTimer RTC_GUARDED_BY(module_thread_checker_);
233 QpParser qp_parser_ RTC_GUARDED_BY(decoder_thread_checker_);
234 ThreadUnsafeOneTimeEvent first_frame_received_
235 RTC_GUARDED_BY(decoder_thread_checker_);
236 // Modified on the construction thread. Can be read without a lock and assumed
237 // to be non-null on the module and decoder threads.
238 ProcessThread* process_thread_ = nullptr;
239 bool is_attached_to_process_thread_
240 RTC_GUARDED_BY(construction_thread_checker_) = false;
241#if RTC_DCHECK_IS_ON
242 bool decoder_thread_is_running_ = false;
243#endif
niklase@google.com470e71d2011-07-07 08:21:25244};
andresp@webrtc.orgf7eb75b2013-09-14 00:25:28245
246} // namespace vcm
pbos@webrtc.orgd900e8b2013-07-03 15:12:26247} // namespace webrtc
Mirko Bonadei92ea95e2017-09-15 04:47:31248#endif // MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_