blob: c45157697bd5ac1656ceaa4cd385ed2446918950 [file] [log] [blame]
Sebastian Jansson8e0b15b2018-04-18 17:19:221/*
2 * Copyright 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10#ifndef VIDEO_VIDEO_SEND_STREAM_IMPL_H_
11#define VIDEO_VIDEO_SEND_STREAM_IMPL_H_
12
Yves Gerey3e707812018-11-28 15:47:4913#include <stddef.h>
14#include <stdint.h>
Sebastian Janssonecb68972019-01-18 09:30:5415#include <atomic>
Sebastian Jansson8e0b15b2018-04-18 17:19:2216#include <map>
17#include <memory>
Sebastian Jansson8e0b15b2018-04-18 17:19:2218#include <vector>
19
Yves Gerey3e707812018-11-28 15:47:4920#include "absl/types/optional.h"
21#include "api/fec_controller.h"
22#include "api/video/encoded_image.h"
23#include "api/video/video_bitrate_allocation.h"
Jiawei Ou4206a0a2018-07-20 22:49:4324#include "api/video/video_bitrate_allocator.h"
Niels Möller213618e2018-07-24 07:29:5825#include "api/video/video_stream_encoder_interface.h"
Yves Gerey3e707812018-11-28 15:47:4926#include "api/video_codecs/video_encoder.h"
27#include "api/video_codecs/video_encoder_config.h"
Sebastian Jansson8e0b15b2018-04-18 17:19:2228#include "call/bitrate_allocator.h"
Yves Gerey3e707812018-11-28 15:47:4929#include "call/rtp_config.h"
30#include "call/rtp_transport_controller_send_interface.h"
Stefan Holmer9416ef82018-07-19 08:34:3831#include "call/rtp_video_sender_interface.h"
Yves Gerey3e707812018-11-28 15:47:4932#include "logging/rtc_event_log/rtc_event_log.h"
33#include "modules/include/module_common_types.h"
34#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
Sebastian Jansson8e0b15b2018-04-18 17:19:2235#include "modules/utility/include/process_thread.h"
Yves Gerey3e707812018-11-28 15:47:4936#include "modules/video_coding/include/video_codec_interface.h"
Steve Anton10542f22019-01-11 17:11:0037#include "rtc_base/critical_section.h"
Yves Gerey3e707812018-11-28 15:47:4938#include "rtc_base/task_queue.h"
Sebastian Janssonecb68972019-01-18 09:30:5439#include "rtc_base/task_utils/repeating_task.h"
Yves Gerey3e707812018-11-28 15:47:4940#include "rtc_base/thread_annotations.h"
Sebastian Jansson8e0b15b2018-04-18 17:19:2241#include "rtc_base/weak_ptr.h"
42#include "video/call_stats.h"
Elad Alon14d1c9d2019-04-08 12:16:1743#include "video/encoder_rtcp_feedback.h"
Sebastian Jansson8e0b15b2018-04-18 17:19:2244#include "video/send_delay_stats.h"
45#include "video/send_statistics_proxy.h"
46#include "video/video_send_stream.h"
Sebastian Jansson8e0b15b2018-04-18 17:19:2247
48namespace webrtc {
49namespace internal {
50
Christoffer Rodbro196c5ba2018-11-27 10:56:2551// Pacing buffer config; overridden by ALR config if provided.
52struct PacingConfig {
53 PacingConfig();
54 PacingConfig(const PacingConfig&);
55 PacingConfig& operator=(const PacingConfig&) = default;
56 ~PacingConfig();
57 FieldTrialParameter<double> pacing_factor;
58 FieldTrialParameter<TimeDelta> max_pacing_delay;
59};
60
Sebastian Jansson8e0b15b2018-04-18 17:19:2261// VideoSendStreamImpl implements internal::VideoSendStream.
62// It is created and destroyed on |worker_queue|. The intent is to decrease the
63// need for locking and to ensure methods are called in sequence.
64// Public methods except |DeliverRtcp| must be called on |worker_queue|.
65// DeliverRtcp is called on the libjingle worker thread or a network thread.
66// An encoder may deliver frames through the EncodedImageCallback on an
67// arbitrary thread.
68class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
Niels Möller213618e2018-07-24 07:29:5869 public VideoStreamEncoderInterface::EncoderSink,
Stefan Holmer64be7fa2018-10-04 13:21:5570 public VideoBitrateAllocationObserver {
Sebastian Jansson8e0b15b2018-04-18 17:19:2271 public:
72 VideoSendStreamImpl(
Sebastian Jansson572c60f2019-03-04 17:30:4173 Clock* clock,
Sebastian Jansson8e0b15b2018-04-18 17:19:2274 SendStatisticsProxy* stats_proxy,
75 rtc::TaskQueue* worker_queue,
76 CallStats* call_stats,
77 RtpTransportControllerSendInterface* transport,
Sebastian Jansson652dc912018-04-19 15:09:1578 BitrateAllocatorInterface* bitrate_allocator,
Sebastian Jansson8e0b15b2018-04-18 17:19:2279 SendDelayStats* send_delay_stats,
Sebastian Jansson652dc912018-04-19 15:09:1580 VideoStreamEncoderInterface* video_stream_encoder,
Sebastian Jansson8e0b15b2018-04-18 17:19:2281 RtcEventLog* event_log,
82 const VideoSendStream::Config* config,
83 int initial_encoder_max_bitrate,
84 double initial_encoder_bitrate_priority,
85 std::map<uint32_t, RtpState> suspended_ssrcs,
86 std::map<uint32_t, RtpPayloadState> suspended_payload_states,
87 VideoEncoderConfig::ContentType content_type,
Niels Möller46879152019-01-07 14:54:4788 std::unique_ptr<FecController> fec_controller,
89 MediaTransportInterface* media_transport);
Sebastian Jansson8e0b15b2018-04-18 17:19:2290 ~VideoSendStreamImpl() override;
91
92 // RegisterProcessThread register |module_process_thread| with those objects
93 // that use it. Registration has to happen on the thread were
94 // |module_process_thread| was created (libjingle's worker thread).
95 // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue,
96 // maybe |worker_queue|.
97 void RegisterProcessThread(ProcessThread* module_process_thread);
98 void DeRegisterProcessThread();
99
Niels Möller8fb1a6a2019-03-05 13:29:42100 void DeliverRtcp(const uint8_t* packet, size_t length);
Sebastian Jansson8e0b15b2018-04-18 17:19:22101 void UpdateActiveSimulcastLayers(const std::vector<bool> active_layers);
102 void Start();
103 void Stop();
104
Stefan Holmerdbdb3a02018-07-17 14:03:46105 // TODO(holmer): Move these to RtpTransportControllerSend.
106 std::map<uint32_t, RtpState> GetRtpStates() const;
107
108 std::map<uint32_t, RtpPayloadState> GetRtpPayloadStates() const;
Sebastian Jansson8e0b15b2018-04-18 17:19:22109
Danil Chapovalovb9b146c2018-06-15 10:28:07110 absl::optional<float> configured_pacing_factor_;
Sebastian Jansson8e0b15b2018-04-18 17:19:22111
Sebastian Jansson8e0b15b2018-04-18 17:19:22112 private:
Sebastian Jansson8e0b15b2018-04-18 17:19:22113 // Implements BitrateAllocatorObserver.
Sebastian Janssonc0e4d452018-10-25 13:08:32114 uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) override;
Sebastian Jansson8e0b15b2018-04-18 17:19:22115
Rasmus Brandtc402dbe2019-02-04 10:09:46116 void OnEncoderConfigurationChanged(
117 std::vector<VideoStream> streams,
118 VideoEncoderConfig::ContentType content_type,
119 int min_transmit_bitrate_bps) override;
Sebastian Jansson8e0b15b2018-04-18 17:19:22120
121 // Implements EncodedImageCallback. The implementation routes encoded frames
122 // to the |payload_router_| and |config.pre_encode_callback| if set.
123 // Called on an arbitrary encoder callback thread.
124 EncodedImageCallback::Result OnEncodedImage(
125 const EncodedImage& encoded_image,
126 const CodecSpecificInfo* codec_specific_info,
127 const RTPFragmentationHeader* fragmentation) override;
128
129 // Implements VideoBitrateAllocationObserver.
Erik Språng566124a2018-04-23 10:32:22130 void OnBitrateAllocationUpdated(
131 const VideoBitrateAllocation& allocation) override;
Sebastian Jansson8e0b15b2018-04-18 17:19:22132
133 // Starts monitoring and sends a keyframe.
134 void StartupVideoSendStream();
135 // Removes the bitrate observer, stops monitoring and notifies the video
136 // encoder of the bitrate update.
Sebastian Janssonecb68972019-01-18 09:30:54137 void StopVideoSendStream() RTC_RUN_ON(worker_queue_);
Sebastian Jansson8e0b15b2018-04-18 17:19:22138
139 void ConfigureProtection();
140 void ConfigureSsrcs();
141 void SignalEncoderTimedOut();
142 void SignalEncoderActive();
Sebastian Jansson464a5572019-02-12 12:32:32143 MediaStreamAllocationConfig GetAllocationConfig() const
144 RTC_RUN_ON(worker_queue_);
Sebastian Jansson572c60f2019-03-04 17:30:41145 Clock* const clock_;
Erik Språngb57ab382018-09-13 08:52:38146 const bool has_alr_probing_;
Christoffer Rodbro196c5ba2018-11-27 10:56:25147 const PacingConfig pacing_config_;
Sebastian Jansson8e0b15b2018-04-18 17:19:22148
149 SendStatisticsProxy* const stats_proxy_;
150 const VideoSendStream::Config* const config_;
Sebastian Jansson8e0b15b2018-04-18 17:19:22151
Sebastian Jansson8e0b15b2018-04-18 17:19:22152 rtc::TaskQueue* const worker_queue_;
153
Sebastian Janssonecb68972019-01-18 09:30:54154 RepeatingTaskHandle check_encoder_activity_task_
155 RTC_GUARDED_BY(worker_queue_);
156
157 std::atomic_bool activity_;
158 bool timed_out_ RTC_GUARDED_BY(worker_queue_);
Sebastian Jansson8e0b15b2018-04-18 17:19:22159
160 CallStats* const call_stats_;
161 RtpTransportControllerSendInterface* const transport_;
Sebastian Jansson652dc912018-04-19 15:09:15162 BitrateAllocatorInterface* const bitrate_allocator_;
Sebastian Jansson8e0b15b2018-04-18 17:19:22163
Sebastian Jansson8e0b15b2018-04-18 17:19:22164 rtc::CriticalSection ivf_writers_crit_;
Sebastian Jansson8e0b15b2018-04-18 17:19:22165
Ilya Nikolaevskiyaa9aa572019-04-11 07:20:24166 bool disable_padding_;
Sebastian Jansson8e0b15b2018-04-18 17:19:22167 int max_padding_bitrate_;
168 int encoder_min_bitrate_bps_;
169 uint32_t encoder_max_bitrate_bps_;
170 uint32_t encoder_target_rate_bps_;
171 double encoder_bitrate_priority_;
172 bool has_packet_feedback_;
173
Sebastian Jansson652dc912018-04-19 15:09:15174 VideoStreamEncoderInterface* const video_stream_encoder_;
Elad Alon14d1c9d2019-04-08 12:16:17175 EncoderRtcpFeedback encoder_feedback_;
Sebastian Jansson8e0b15b2018-04-18 17:19:22176
177 RtcpBandwidthObserver* const bandwidth_observer_;
Stefan Holmer9416ef82018-07-19 08:34:38178 RtpVideoSenderInterface* const rtp_video_sender_;
Sebastian Jansson8e0b15b2018-04-18 17:19:22179
180 // |weak_ptr_| to our self. This is used since we can not call
181 // |weak_ptr_factory_.GetWeakPtr| from multiple sequences but it is ok to copy
182 // an existing WeakPtr.
183 rtc::WeakPtr<VideoSendStreamImpl> weak_ptr_;
184 // |weak_ptr_factory_| must be declared last to make sure all WeakPtr's are
185 // invalidated before any other members are destroyed.
186 rtc::WeakPtrFactory<VideoSendStreamImpl> weak_ptr_factory_;
187
Erik Språng4e193e42018-09-14 17:01:58188 // Context for the most recent and last sent video bitrate allocation. Used to
189 // throttle sending of similar bitrate allocations.
190 struct VbaSendContext {
191 VideoBitrateAllocation last_sent_allocation;
192 absl::optional<VideoBitrateAllocation> throttled_allocation;
193 int64_t last_send_time_ms;
194 };
195 absl::optional<VbaSendContext> video_bitrate_allocation_context_
196 RTC_GUARDED_BY(worker_queue_);
Niels Möller46879152019-01-07 14:54:47197 MediaTransportInterface* const media_transport_;
198 rtc::CriticalSection media_transport_id_lock_;
199 int64_t media_transport_frame_id_ RTC_GUARDED_BY(media_transport_id_lock_) =
200 0;
Sebastian Jansson8e0b15b2018-04-18 17:19:22201};
202} // namespace internal
203} // namespace webrtc
204#endif // VIDEO_VIDEO_SEND_STREAM_IMPL_H_