Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | #ifndef VIDEO_VIDEO_SEND_STREAM_IMPL_H_ |
| 11 | #define VIDEO_VIDEO_SEND_STREAM_IMPL_H_ |
| 12 | |
Yves Gerey | 3e70781 | 2018-11-28 15:47:49 | [diff] [blame] | 13 | #include <stddef.h> |
| 14 | #include <stdint.h> |
Sebastian Jansson | ecb6897 | 2019-01-18 09:30:54 | [diff] [blame] | 15 | #include <atomic> |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 16 | #include <map> |
| 17 | #include <memory> |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 18 | #include <vector> |
| 19 | |
Yves Gerey | 3e70781 | 2018-11-28 15:47:49 | [diff] [blame] | 20 | #include "absl/types/optional.h" |
| 21 | #include "api/fec_controller.h" |
| 22 | #include "api/video/encoded_image.h" |
| 23 | #include "api/video/video_bitrate_allocation.h" |
Jiawei Ou | 4206a0a | 2018-07-20 22:49:43 | [diff] [blame] | 24 | #include "api/video/video_bitrate_allocator.h" |
Niels Möller | 213618e | 2018-07-24 07:29:58 | [diff] [blame] | 25 | #include "api/video/video_stream_encoder_interface.h" |
Yves Gerey | 3e70781 | 2018-11-28 15:47:49 | [diff] [blame] | 26 | #include "api/video_codecs/video_encoder.h" |
| 27 | #include "api/video_codecs/video_encoder_config.h" |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 28 | #include "call/bitrate_allocator.h" |
Yves Gerey | 3e70781 | 2018-11-28 15:47:49 | [diff] [blame] | 29 | #include "call/rtp_config.h" |
| 30 | #include "call/rtp_transport_controller_send_interface.h" |
Stefan Holmer | 9416ef8 | 2018-07-19 08:34:38 | [diff] [blame] | 31 | #include "call/rtp_video_sender_interface.h" |
Yves Gerey | 3e70781 | 2018-11-28 15:47:49 | [diff] [blame] | 32 | #include "logging/rtc_event_log/rtc_event_log.h" |
| 33 | #include "modules/include/module_common_types.h" |
| 34 | #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 35 | #include "modules/utility/include/process_thread.h" |
Yves Gerey | 3e70781 | 2018-11-28 15:47:49 | [diff] [blame] | 36 | #include "modules/video_coding/include/video_codec_interface.h" |
Steve Anton | 10542f2 | 2019-01-11 17:11:00 | [diff] [blame] | 37 | #include "rtc_base/critical_section.h" |
Yves Gerey | 3e70781 | 2018-11-28 15:47:49 | [diff] [blame] | 38 | #include "rtc_base/task_queue.h" |
Sebastian Jansson | ecb6897 | 2019-01-18 09:30:54 | [diff] [blame] | 39 | #include "rtc_base/task_utils/repeating_task.h" |
Yves Gerey | 3e70781 | 2018-11-28 15:47:49 | [diff] [blame] | 40 | #include "rtc_base/thread_annotations.h" |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 41 | #include "rtc_base/weak_ptr.h" |
| 42 | #include "video/call_stats.h" |
Elad Alon | 14d1c9d | 2019-04-08 12:16:17 | [diff] [blame] | 43 | #include "video/encoder_rtcp_feedback.h" |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 44 | #include "video/send_delay_stats.h" |
| 45 | #include "video/send_statistics_proxy.h" |
| 46 | #include "video/video_send_stream.h" |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 47 | |
| 48 | namespace webrtc { |
| 49 | namespace internal { |
| 50 | |
Christoffer Rodbro | 196c5ba | 2018-11-27 10:56:25 | [diff] [blame] | 51 | // Pacing buffer config; overridden by ALR config if provided. |
| 52 | struct PacingConfig { |
| 53 | PacingConfig(); |
| 54 | PacingConfig(const PacingConfig&); |
| 55 | PacingConfig& operator=(const PacingConfig&) = default; |
| 56 | ~PacingConfig(); |
| 57 | FieldTrialParameter<double> pacing_factor; |
| 58 | FieldTrialParameter<TimeDelta> max_pacing_delay; |
| 59 | }; |
| 60 | |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 61 | // VideoSendStreamImpl implements internal::VideoSendStream. |
| 62 | // It is created and destroyed on |worker_queue|. The intent is to decrease the |
| 63 | // need for locking and to ensure methods are called in sequence. |
| 64 | // Public methods except |DeliverRtcp| must be called on |worker_queue|. |
| 65 | // DeliverRtcp is called on the libjingle worker thread or a network thread. |
| 66 | // An encoder may deliver frames through the EncodedImageCallback on an |
| 67 | // arbitrary thread. |
| 68 | class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, |
Niels Möller | 213618e | 2018-07-24 07:29:58 | [diff] [blame] | 69 | public VideoStreamEncoderInterface::EncoderSink, |
Stefan Holmer | 64be7fa | 2018-10-04 13:21:55 | [diff] [blame] | 70 | public VideoBitrateAllocationObserver { |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 71 | public: |
| 72 | VideoSendStreamImpl( |
Sebastian Jansson | 572c60f | 2019-03-04 17:30:41 | [diff] [blame] | 73 | Clock* clock, |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 74 | SendStatisticsProxy* stats_proxy, |
| 75 | rtc::TaskQueue* worker_queue, |
| 76 | CallStats* call_stats, |
| 77 | RtpTransportControllerSendInterface* transport, |
Sebastian Jansson | 652dc91 | 2018-04-19 15:09:15 | [diff] [blame] | 78 | BitrateAllocatorInterface* bitrate_allocator, |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 79 | SendDelayStats* send_delay_stats, |
Sebastian Jansson | 652dc91 | 2018-04-19 15:09:15 | [diff] [blame] | 80 | VideoStreamEncoderInterface* video_stream_encoder, |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 81 | RtcEventLog* event_log, |
| 82 | const VideoSendStream::Config* config, |
| 83 | int initial_encoder_max_bitrate, |
| 84 | double initial_encoder_bitrate_priority, |
| 85 | std::map<uint32_t, RtpState> suspended_ssrcs, |
| 86 | std::map<uint32_t, RtpPayloadState> suspended_payload_states, |
| 87 | VideoEncoderConfig::ContentType content_type, |
Niels Möller | 4687915 | 2019-01-07 14:54:47 | [diff] [blame] | 88 | std::unique_ptr<FecController> fec_controller, |
| 89 | MediaTransportInterface* media_transport); |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 90 | ~VideoSendStreamImpl() override; |
| 91 | |
| 92 | // RegisterProcessThread register |module_process_thread| with those objects |
| 93 | // that use it. Registration has to happen on the thread were |
| 94 | // |module_process_thread| was created (libjingle's worker thread). |
| 95 | // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue, |
| 96 | // maybe |worker_queue|. |
| 97 | void RegisterProcessThread(ProcessThread* module_process_thread); |
| 98 | void DeRegisterProcessThread(); |
| 99 | |
Niels Möller | 8fb1a6a | 2019-03-05 13:29:42 | [diff] [blame] | 100 | void DeliverRtcp(const uint8_t* packet, size_t length); |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 101 | void UpdateActiveSimulcastLayers(const std::vector<bool> active_layers); |
| 102 | void Start(); |
| 103 | void Stop(); |
| 104 | |
Stefan Holmer | dbdb3a0 | 2018-07-17 14:03:46 | [diff] [blame] | 105 | // TODO(holmer): Move these to RtpTransportControllerSend. |
| 106 | std::map<uint32_t, RtpState> GetRtpStates() const; |
| 107 | |
| 108 | std::map<uint32_t, RtpPayloadState> GetRtpPayloadStates() const; |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 109 | |
Danil Chapovalov | b9b146c | 2018-06-15 10:28:07 | [diff] [blame] | 110 | absl::optional<float> configured_pacing_factor_; |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 111 | |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 112 | private: |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 113 | // Implements BitrateAllocatorObserver. |
Sebastian Jansson | c0e4d45 | 2018-10-25 13:08:32 | [diff] [blame] | 114 | uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) override; |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 115 | |
Rasmus Brandt | c402dbe | 2019-02-04 10:09:46 | [diff] [blame] | 116 | void OnEncoderConfigurationChanged( |
| 117 | std::vector<VideoStream> streams, |
| 118 | VideoEncoderConfig::ContentType content_type, |
| 119 | int min_transmit_bitrate_bps) override; |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 120 | |
| 121 | // Implements EncodedImageCallback. The implementation routes encoded frames |
| 122 | // to the |payload_router_| and |config.pre_encode_callback| if set. |
| 123 | // Called on an arbitrary encoder callback thread. |
| 124 | EncodedImageCallback::Result OnEncodedImage( |
| 125 | const EncodedImage& encoded_image, |
| 126 | const CodecSpecificInfo* codec_specific_info, |
| 127 | const RTPFragmentationHeader* fragmentation) override; |
| 128 | |
| 129 | // Implements VideoBitrateAllocationObserver. |
Erik Språng | 566124a | 2018-04-23 10:32:22 | [diff] [blame] | 130 | void OnBitrateAllocationUpdated( |
| 131 | const VideoBitrateAllocation& allocation) override; |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 132 | |
| 133 | // Starts monitoring and sends a keyframe. |
| 134 | void StartupVideoSendStream(); |
| 135 | // Removes the bitrate observer, stops monitoring and notifies the video |
| 136 | // encoder of the bitrate update. |
Sebastian Jansson | ecb6897 | 2019-01-18 09:30:54 | [diff] [blame] | 137 | void StopVideoSendStream() RTC_RUN_ON(worker_queue_); |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 138 | |
| 139 | void ConfigureProtection(); |
| 140 | void ConfigureSsrcs(); |
| 141 | void SignalEncoderTimedOut(); |
| 142 | void SignalEncoderActive(); |
Sebastian Jansson | 464a557 | 2019-02-12 12:32:32 | [diff] [blame] | 143 | MediaStreamAllocationConfig GetAllocationConfig() const |
| 144 | RTC_RUN_ON(worker_queue_); |
Sebastian Jansson | 572c60f | 2019-03-04 17:30:41 | [diff] [blame] | 145 | Clock* const clock_; |
Erik Språng | b57ab38 | 2018-09-13 08:52:38 | [diff] [blame] | 146 | const bool has_alr_probing_; |
Christoffer Rodbro | 196c5ba | 2018-11-27 10:56:25 | [diff] [blame] | 147 | const PacingConfig pacing_config_; |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 148 | |
| 149 | SendStatisticsProxy* const stats_proxy_; |
| 150 | const VideoSendStream::Config* const config_; |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 151 | |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 152 | rtc::TaskQueue* const worker_queue_; |
| 153 | |
Sebastian Jansson | ecb6897 | 2019-01-18 09:30:54 | [diff] [blame] | 154 | RepeatingTaskHandle check_encoder_activity_task_ |
| 155 | RTC_GUARDED_BY(worker_queue_); |
| 156 | |
| 157 | std::atomic_bool activity_; |
| 158 | bool timed_out_ RTC_GUARDED_BY(worker_queue_); |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 159 | |
| 160 | CallStats* const call_stats_; |
| 161 | RtpTransportControllerSendInterface* const transport_; |
Sebastian Jansson | 652dc91 | 2018-04-19 15:09:15 | [diff] [blame] | 162 | BitrateAllocatorInterface* const bitrate_allocator_; |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 163 | |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 164 | rtc::CriticalSection ivf_writers_crit_; |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 165 | |
Ilya Nikolaevskiy | aa9aa57 | 2019-04-11 07:20:24 | [diff] [blame] | 166 | bool disable_padding_; |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 167 | int max_padding_bitrate_; |
| 168 | int encoder_min_bitrate_bps_; |
| 169 | uint32_t encoder_max_bitrate_bps_; |
| 170 | uint32_t encoder_target_rate_bps_; |
| 171 | double encoder_bitrate_priority_; |
| 172 | bool has_packet_feedback_; |
| 173 | |
Sebastian Jansson | 652dc91 | 2018-04-19 15:09:15 | [diff] [blame] | 174 | VideoStreamEncoderInterface* const video_stream_encoder_; |
Elad Alon | 14d1c9d | 2019-04-08 12:16:17 | [diff] [blame] | 175 | EncoderRtcpFeedback encoder_feedback_; |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 176 | |
| 177 | RtcpBandwidthObserver* const bandwidth_observer_; |
Stefan Holmer | 9416ef8 | 2018-07-19 08:34:38 | [diff] [blame] | 178 | RtpVideoSenderInterface* const rtp_video_sender_; |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 179 | |
| 180 | // |weak_ptr_| to our self. This is used since we can not call |
| 181 | // |weak_ptr_factory_.GetWeakPtr| from multiple sequences but it is ok to copy |
| 182 | // an existing WeakPtr. |
| 183 | rtc::WeakPtr<VideoSendStreamImpl> weak_ptr_; |
| 184 | // |weak_ptr_factory_| must be declared last to make sure all WeakPtr's are |
| 185 | // invalidated before any other members are destroyed. |
| 186 | rtc::WeakPtrFactory<VideoSendStreamImpl> weak_ptr_factory_; |
| 187 | |
Erik Språng | 4e193e4 | 2018-09-14 17:01:58 | [diff] [blame] | 188 | // Context for the most recent and last sent video bitrate allocation. Used to |
| 189 | // throttle sending of similar bitrate allocations. |
| 190 | struct VbaSendContext { |
| 191 | VideoBitrateAllocation last_sent_allocation; |
| 192 | absl::optional<VideoBitrateAllocation> throttled_allocation; |
| 193 | int64_t last_send_time_ms; |
| 194 | }; |
| 195 | absl::optional<VbaSendContext> video_bitrate_allocation_context_ |
| 196 | RTC_GUARDED_BY(worker_queue_); |
Niels Möller | 4687915 | 2019-01-07 14:54:47 | [diff] [blame] | 197 | MediaTransportInterface* const media_transport_; |
| 198 | rtc::CriticalSection media_transport_id_lock_; |
| 199 | int64_t media_transport_frame_id_ RTC_GUARDED_BY(media_transport_id_lock_) = |
| 200 | 0; |
Sebastian Jansson | 8e0b15b | 2018-04-18 17:19:22 | [diff] [blame] | 201 | }; |
| 202 | } // namespace internal |
| 203 | } // namespace webrtc |
| 204 | #endif // VIDEO_VIDEO_SEND_STREAM_IMPL_H_ |