blob: 862dfde14c1392da46b8f93223e78820a1045257 [file] [log] [blame]
Sebastian Jansson8e0b15b2018-04-18 17:19:221/*
2 * Copyright 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10#include "video/video_send_stream_impl.h"
11
Yves Gerey3e707812018-11-28 15:47:4912#include <stdio.h>
Jonas Olssona4d87372019-07-05 17:08:3313
Sebastian Jansson8e0b15b2018-04-18 17:19:2214#include <algorithm>
Yves Gerey3e707812018-11-28 15:47:4915#include <cstdint>
Sebastian Jansson8e0b15b2018-04-18 17:19:2216#include <string>
17#include <utility>
18
Steve Antonbd631a02019-03-28 17:51:2719#include "absl/algorithm/container.h"
Steve Anton10542f22019-01-11 17:11:0020#include "api/crypto/crypto_options.h"
21#include "api/rtp_parameters.h"
Mirko Bonadeid9708072019-01-25 19:26:4822#include "api/scoped_refptr.h"
Artem Titovd15a5752021-02-10 13:31:2423#include "api/sequence_checker.h"
Yves Gerey3e707812018-11-28 15:47:4924#include "api/video_codecs/video_codec.h"
Sebastian Jansson8e0b15b2018-04-18 17:19:2225#include "call/rtp_transport_controller_send_interface.h"
Yves Gerey3e707812018-11-28 15:47:4926#include "call/video_send_stream.h"
Yves Gerey3e707812018-11-28 15:47:4927#include "modules/pacing/paced_sender.h"
Steve Anton10542f22019-01-11 17:11:0028#include "rtc_base/atomic_ops.h"
Sebastian Jansson8e0b15b2018-04-18 17:19:2229#include "rtc_base/checks.h"
30#include "rtc_base/experiments/alr_experiment.h"
Ying Wang8c5520c2019-09-03 15:25:2131#include "rtc_base/experiments/field_trial_parser.h"
Elad Alon80f53b72019-10-11 14:19:4332#include "rtc_base/experiments/min_video_bitrate_experiment.h"
Erik Språngcd76eab2019-01-21 17:06:4633#include "rtc_base/experiments/rate_control_settings.h"
Sebastian Jansson8e0b15b2018-04-18 17:19:2234#include "rtc_base/logging.h"
35#include "rtc_base/numerics/safe_conversions.h"
Tommi1050fbc2021-06-03 15:58:2836#include "rtc_base/task_utils/to_queued_task.h"
Sebastian Jansson8e0b15b2018-04-18 17:19:2237#include "rtc_base/trace_event.h"
Yves Gerey3e707812018-11-28 15:47:4938#include "system_wrappers/include/clock.h"
Sebastian Jansson8e0b15b2018-04-18 17:19:2239#include "system_wrappers/include/field_trial.h"
40
41namespace webrtc {
42namespace internal {
43namespace {
Sebastian Jansson8e0b15b2018-04-18 17:19:2244
Erik Språng4e193e42018-09-14 17:01:5845// Max positive size difference to treat allocations as "similar".
46static constexpr int kMaxVbaSizeDifferencePercent = 10;
47// Max time we will throttle similar video bitrate allocations.
48static constexpr int64_t kMaxVbaThrottleTimeMs = 500;
49
Danil Chapovalov0c626af2020-02-10 10:16:0050constexpr TimeDelta kEncoderTimeOut = TimeDelta::Seconds(2);
Sebastian Janssonecb68972019-01-18 09:30:5451
Erik Språng9d69cbe2020-10-22 15:44:4252// When send-side BWE is used a stricter 1.1x pacing factor is used, rather than
53// the 2.5x which is used with receive-side BWE. Provides a more careful
54// bandwidth rampup with less risk of overshoots causing adverse effects like
55// packet loss. Not used for receive side BWE, since there we lack the probing
56// feature and so may result in too slow initial rampup.
57static constexpr double kStrictPacingMultiplier = 1.1;
58
Sebastian Jansson8e0b15b2018-04-18 17:19:2259bool TransportSeqNumExtensionConfigured(const VideoSendStream::Config& config) {
60 const std::vector<RtpExtension>& extensions = config.rtp.extensions;
Steve Antonbd631a02019-03-28 17:51:2761 return absl::c_any_of(extensions, [](const RtpExtension& ext) {
62 return ext.uri == RtpExtension::kTransportSequenceNumberUri;
63 });
Sebastian Jansson8e0b15b2018-04-18 17:19:2264}
65
Erik Språngb57ab382018-09-13 08:52:3866// Calculate max padding bitrate for a multi layer codec.
67int CalculateMaxPadBitrateBps(const std::vector<VideoStream>& streams,
Ilya Nikolaevskiy93be66c2020-04-02 12:10:2768 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 10:09:4669 VideoEncoderConfig::ContentType content_type,
Sebastian Jansson8e0b15b2018-04-18 17:19:2270 int min_transmit_bitrate_bps,
Erik Språngb57ab382018-09-13 08:52:3871 bool pad_to_min_bitrate,
72 bool alr_probing) {
Sebastian Jansson8e0b15b2018-04-18 17:19:2273 int pad_up_to_bitrate_bps = 0;
Erik Språngb57ab382018-09-13 08:52:3874
Ilya Nikolaevskiy93be66c2020-04-02 12:10:2775 RTC_DCHECK(!is_svc || streams.size() <= 1) << "Only one stream is allowed in "
76 "SVC mode.";
77
Erik Språngb57ab382018-09-13 08:52:3878 // Filter out only the active streams;
79 std::vector<VideoStream> active_streams;
80 for (const VideoStream& stream : streams) {
81 if (stream.active)
82 active_streams.emplace_back(stream);
83 }
84
Ilya Nikolaevskiy93be66c2020-04-02 12:10:2785 if (active_streams.size() > 1 || (!active_streams.empty() && is_svc)) {
86 // Simulcast or SVC is used.
87 // if SVC is used, stream bitrates should already encode svc bitrates:
88 // min_bitrate = min bitrate of a lowest svc layer.
89 // target_bitrate = sum of target bitrates of lower layers + min bitrate
90 // of the last one (as used in the calculations below).
91 // max_bitrate = sum of all active layers' max_bitrate.
Erik Språngb57ab382018-09-13 08:52:3892 if (alr_probing) {
93 // With alr probing, just pad to the min bitrate of the lowest stream,
94 // probing will handle the rest of the rampup.
95 pad_up_to_bitrate_bps = active_streams[0].min_bitrate_bps;
96 } else {
Rasmus Brandtc402dbe2019-02-04 10:09:4697 // Without alr probing, pad up to start bitrate of the
98 // highest active stream.
99 const double hysteresis_factor =
100 RateControlSettings::ParseFromFieldTrials()
101 .GetSimulcastHysteresisFactor(content_type);
Erik Språng576db1b2020-06-08 11:32:20102 if (is_svc) {
103 // For SVC, since there is only one "stream", the padding bitrate
104 // needed to enable the top spatial layer is stored in the
Artem Titovab30d722021-07-27 14:22:11105 // `target_bitrate_bps` field.
Erik Språng576db1b2020-06-08 11:32:20106 // TODO(sprang): This behavior needs to die.
107 pad_up_to_bitrate_bps = static_cast<int>(
108 hysteresis_factor * active_streams[0].target_bitrate_bps + 0.5);
109 } else {
110 const size_t top_active_stream_idx = active_streams.size() - 1;
111 pad_up_to_bitrate_bps = std::min(
112 static_cast<int>(
113 hysteresis_factor *
114 active_streams[top_active_stream_idx].min_bitrate_bps +
115 0.5),
116 active_streams[top_active_stream_idx].target_bitrate_bps);
Rasmus Brandtc402dbe2019-02-04 10:09:46117
Erik Språng576db1b2020-06-08 11:32:20118 // Add target_bitrate_bps of the lower active streams.
119 for (size_t i = 0; i < top_active_stream_idx; ++i) {
120 pad_up_to_bitrate_bps += active_streams[i].target_bitrate_bps;
121 }
Rasmus Brandtc402dbe2019-02-04 10:09:46122 }
Erik Språngb57ab382018-09-13 08:52:38123 }
124 } else if (!active_streams.empty() && pad_to_min_bitrate) {
125 pad_up_to_bitrate_bps = active_streams[0].min_bitrate_bps;
Sebastian Jansson8e0b15b2018-04-18 17:19:22126 }
127
128 pad_up_to_bitrate_bps =
129 std::max(pad_up_to_bitrate_bps, min_transmit_bitrate_bps);
130
131 return pad_up_to_bitrate_bps;
132}
133
Erik Språngb57ab382018-09-13 08:52:38134absl::optional<AlrExperimentSettings> GetAlrSettings(
135 VideoEncoderConfig::ContentType content_type) {
136 if (content_type == VideoEncoderConfig::ContentType::kScreen) {
137 return AlrExperimentSettings::CreateFromFieldTrial(
138 AlrExperimentSettings::kScreenshareProbingBweExperimentName);
139 }
140 return AlrExperimentSettings::CreateFromFieldTrial(
141 AlrExperimentSettings::kStrictPacingAndProbingExperimentName);
142}
Erik Språng4e193e42018-09-14 17:01:58143
144bool SameStreamsEnabled(const VideoBitrateAllocation& lhs,
145 const VideoBitrateAllocation& rhs) {
146 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
147 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
148 if (lhs.HasBitrate(si, ti) != rhs.HasBitrate(si, ti)) {
149 return false;
150 }
151 }
152 }
153 return true;
154}
Tommie902f282021-06-03 10:02:02155
156// Returns an optional that has value iff TransportSeqNumExtensionConfigured
157// is `true` for the given video send stream config.
158absl::optional<float> GetConfiguredPacingFactor(
159 const VideoSendStream::Config& config,
160 VideoEncoderConfig::ContentType content_type,
161 const PacingConfig& default_pacing_config) {
162 if (!TransportSeqNumExtensionConfigured(config))
163 return absl::nullopt;
164
165 absl::optional<AlrExperimentSettings> alr_settings =
166 GetAlrSettings(content_type);
167 if (alr_settings)
168 return alr_settings->pacing_factor;
169
170 RateControlSettings rate_control_settings =
171 RateControlSettings::ParseFromFieldTrials();
172 return rate_control_settings.GetPacingFactor().value_or(
173 default_pacing_config.pacing_factor);
174}
175
Tommifa3ce632021-06-03 10:06:02176uint32_t GetInitialEncoderMaxBitrate(int initial_encoder_max_bitrate) {
177 if (initial_encoder_max_bitrate > 0)
178 return rtc::dchecked_cast<uint32_t>(initial_encoder_max_bitrate);
179
180 // TODO(srte): Make sure max bitrate is not set to negative values. We don't
181 // have any way to handle unset values in downstream code, such as the
182 // bitrate allocator. Previously -1 was implicitly casted to UINT32_MAX, a
183 // behaviour that is not safe. Converting to 10 Mbps should be safe for
184 // reasonable use cases as it allows adding the max of multiple streams
185 // without wrappping around.
186 const int kFallbackMaxBitrateBps = 10000000;
187 RTC_DLOG(LS_ERROR) << "ERROR: Initial encoder max bitrate = "
188 << initial_encoder_max_bitrate << " which is <= 0!";
189 RTC_DLOG(LS_INFO) << "Using default encoder max bitrate = 10 Mbps";
190 return kFallbackMaxBitrateBps;
191}
192
Sebastian Jansson8e0b15b2018-04-18 17:19:22193} // namespace
194
Christoffer Rodbro196c5ba2018-11-27 10:56:25195PacingConfig::PacingConfig()
Erik Språng9d69cbe2020-10-22 15:44:42196 : pacing_factor("factor", kStrictPacingMultiplier),
Christoffer Rodbro196c5ba2018-11-27 10:56:25197 max_pacing_delay("max_delay",
Danil Chapovalov0c626af2020-02-10 10:16:00198 TimeDelta::Millis(PacedSender::kMaxQueueLengthMs)) {
Christoffer Rodbro196c5ba2018-11-27 10:56:25199 ParseFieldTrial({&pacing_factor, &max_pacing_delay},
200 field_trial::FindFullName("WebRTC-Video-Pacing"));
201}
202PacingConfig::PacingConfig(const PacingConfig&) = default;
203PacingConfig::~PacingConfig() = default;
204
Sebastian Jansson8e0b15b2018-04-18 17:19:22205VideoSendStreamImpl::VideoSendStreamImpl(
Sebastian Jansson572c60f2019-03-04 17:30:41206 Clock* clock,
Sebastian Jansson8e0b15b2018-04-18 17:19:22207 SendStatisticsProxy* stats_proxy,
Tommifa3ce632021-06-03 10:06:02208 rtc::TaskQueue* rtp_transport_queue,
Sebastian Jansson8e0b15b2018-04-18 17:19:22209 RtpTransportControllerSendInterface* transport,
Sebastian Jansson652dc912018-04-19 15:09:15210 BitrateAllocatorInterface* bitrate_allocator,
Sebastian Jansson652dc912018-04-19 15:09:15211 VideoStreamEncoderInterface* video_stream_encoder,
Sebastian Jansson8e0b15b2018-04-18 17:19:22212 const VideoSendStream::Config* config,
213 int initial_encoder_max_bitrate,
214 double initial_encoder_bitrate_priority,
Sebastian Jansson8e0b15b2018-04-18 17:19:22215 VideoEncoderConfig::ContentType content_type,
Tommi1050fbc2021-06-03 15:58:28216 RtpVideoSenderInterface* rtp_video_sender)
Sebastian Jansson572c60f2019-03-04 17:30:41217 : clock_(clock),
218 has_alr_probing_(config->periodic_alr_bandwidth_probing ||
Erik Språngb57ab382018-09-13 08:52:38219 GetAlrSettings(content_type)),
Christoffer Rodbro196c5ba2018-11-27 10:56:25220 pacing_config_(PacingConfig()),
Sebastian Jansson8e0b15b2018-04-18 17:19:22221 stats_proxy_(stats_proxy),
222 config_(config),
Tommifa3ce632021-06-03 10:06:02223 rtp_transport_queue_(rtp_transport_queue),
Erik Språngcd76eab2019-01-21 17:06:46224 timed_out_(false),
Sebastian Jansson8e0b15b2018-04-18 17:19:22225 transport_(transport),
226 bitrate_allocator_(bitrate_allocator),
Ilya Nikolaevskiyaa9aa572019-04-11 07:20:24227 disable_padding_(true),
Sebastian Jansson8e0b15b2018-04-18 17:19:22228 max_padding_bitrate_(0),
229 encoder_min_bitrate_bps_(0),
Tommifa3ce632021-06-03 10:06:02230 encoder_max_bitrate_bps_(
231 GetInitialEncoderMaxBitrate(initial_encoder_max_bitrate)),
Sebastian Jansson8e0b15b2018-04-18 17:19:22232 encoder_target_rate_bps_(0),
233 encoder_bitrate_priority_(initial_encoder_bitrate_priority),
Sebastian Jansson8e0b15b2018-04-18 17:19:22234 video_stream_encoder_(video_stream_encoder),
Sebastian Jansson8e0b15b2018-04-18 17:19:22235 bandwidth_observer_(transport->GetBandwidthObserver()),
Tommi1050fbc2021-06-03 15:58:28236 rtp_video_sender_(rtp_video_sender),
Tommie902f282021-06-03 10:02:02237 configured_pacing_factor_(
238 GetConfiguredPacingFactor(*config_, content_type, pacing_config_)) {
Tommifa3ce632021-06-03 10:06:02239 RTC_DCHECK_GE(config_->rtp.payload_type, 0);
240 RTC_DCHECK_LE(config_->rtp.payload_type, 127);
Bjorn A Mellem7a9a0922019-11-26 17:19:40241 RTC_DCHECK(!config_->rtp.ssrcs.empty());
Sebastian Jansson8e0b15b2018-04-18 17:19:22242 RTC_DCHECK(transport_);
243 RTC_DCHECK_NE(initial_encoder_max_bitrate, 0);
Tommi1050fbc2021-06-03 15:58:28244 RTC_LOG(LS_INFO) << "VideoSendStreamImpl: " << config_->ToString();
Sebastian Jansson8e0b15b2018-04-18 17:19:22245
246 RTC_CHECK(AlrExperimentSettings::MaxOneFieldTrialEnabled());
Tommifa3ce632021-06-03 10:06:02247
Tommi1050fbc2021-06-03 15:58:28248 // Only request rotation at the source when we positively know that the remote
249 // side doesn't support the rotation extension. This allows us to prepare the
250 // encoder in the expectation that rotation is supported - which is the common
251 // case.
252 bool rotation_applied = absl::c_none_of(
253 config_->rtp.extensions, [](const RtpExtension& extension) {
254 return extension.uri == RtpExtension::kVideoRotationUri;
255 });
256
257 video_stream_encoder_->SetSink(this, rotation_applied);
Tommifa3ce632021-06-03 10:06:02258
259 absl::optional<bool> enable_alr_bw_probing;
260
Sebastian Jansson8e0b15b2018-04-18 17:19:22261 // If send-side BWE is enabled, check if we should apply updated probing and
262 // pacing settings.
Tommi1050fbc2021-06-03 15:58:28263 if (configured_pacing_factor_) {
Erik Språngb57ab382018-09-13 08:52:38264 absl::optional<AlrExperimentSettings> alr_settings =
265 GetAlrSettings(content_type);
Tommifa3ce632021-06-03 10:06:02266 int queue_time_limit_ms;
Sebastian Jansson8e0b15b2018-04-18 17:19:22267 if (alr_settings) {
Tommifa3ce632021-06-03 10:06:02268 enable_alr_bw_probing = true;
269 queue_time_limit_ms = alr_settings->max_paced_queue_time;
Sebastian Jansson8e0b15b2018-04-18 17:19:22270 } else {
Erik Språngcd76eab2019-01-21 17:06:46271 RateControlSettings rate_control_settings =
272 RateControlSettings::ParseFromFieldTrials();
Tommifa3ce632021-06-03 10:06:02273 enable_alr_bw_probing = rate_control_settings.UseAlrProbing();
274 queue_time_limit_ms = pacing_config_.max_pacing_delay.Get().ms();
Sebastian Jansson8e0b15b2018-04-18 17:19:22275 }
Tommie902f282021-06-03 10:02:02276
Tommifa3ce632021-06-03 10:06:02277 transport->SetQueueTimeLimit(queue_time_limit_ms);
Sebastian Jansson8e0b15b2018-04-18 17:19:22278 }
279
280 if (config_->periodic_alr_bandwidth_probing) {
Tommifa3ce632021-06-03 10:06:02281 enable_alr_bw_probing = config_->periodic_alr_bandwidth_probing;
Sebastian Jansson8e0b15b2018-04-18 17:19:22282 }
283
Tommifa3ce632021-06-03 10:06:02284 if (enable_alr_bw_probing) {
285 transport->EnablePeriodicAlrProbing(*enable_alr_bw_probing);
286 }
Sebastian Jansson8e0b15b2018-04-18 17:19:22287
Tommi1050fbc2021-06-03 15:58:28288 rtp_transport_queue_->PostTask(ToQueuedTask(transport_queue_safety_, [this] {
289 if (configured_pacing_factor_)
290 transport_->SetPacingFactor(*configured_pacing_factor_);
291
292 video_stream_encoder_->SetStartBitrate(
293 bitrate_allocator_->GetStartBitrate(this));
294 }));
Sebastian Jansson8e0b15b2018-04-18 17:19:22295}
296
Sebastian Jansson8e0b15b2018-04-18 17:19:22297VideoSendStreamImpl::~VideoSendStreamImpl() {
Tommi1050fbc2021-06-03 15:58:28298 RTC_DCHECK_RUN_ON(&thread_checker_);
299 RTC_LOG(LS_INFO) << "~VideoSendStreamImpl: " << config_->ToString();
Sebastian Jansson8e0b15b2018-04-18 17:19:22300}
301
Niels Möller8fb1a6a2019-03-05 13:29:42302void VideoSendStreamImpl::DeliverRtcp(const uint8_t* packet, size_t length) {
Sebastian Jansson8e0b15b2018-04-18 17:19:22303 // Runs on a network thread.
Tommifa3ce632021-06-03 10:06:02304 RTC_DCHECK(!rtp_transport_queue_->IsCurrent());
Stefan Holmer9416ef82018-07-19 08:34:38305 rtp_video_sender_->DeliverRtcp(packet, length);
Sebastian Jansson8e0b15b2018-04-18 17:19:22306}
307
308void VideoSendStreamImpl::UpdateActiveSimulcastLayers(
309 const std::vector<bool> active_layers) {
Tommifa3ce632021-06-03 10:06:02310 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Stefan Holmer9416ef82018-07-19 08:34:38311 bool previously_active = rtp_video_sender_->IsActive();
312 rtp_video_sender_->SetActiveModules(active_layers);
313 if (!rtp_video_sender_->IsActive() && previously_active) {
Sebastian Jansson8e0b15b2018-04-18 17:19:22314 // Payload router switched from active to inactive.
315 StopVideoSendStream();
Stefan Holmer9416ef82018-07-19 08:34:38316 } else if (rtp_video_sender_->IsActive() && !previously_active) {
Sebastian Jansson8e0b15b2018-04-18 17:19:22317 // Payload router switched from inactive to active.
318 StartupVideoSendStream();
319 }
320}
321
322void VideoSendStreamImpl::Start() {
Tommifa3ce632021-06-03 10:06:02323 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Sebastian Jansson8e0b15b2018-04-18 17:19:22324 RTC_LOG(LS_INFO) << "VideoSendStream::Start";
Stefan Holmer9416ef82018-07-19 08:34:38325 if (rtp_video_sender_->IsActive())
Sebastian Jansson8e0b15b2018-04-18 17:19:22326 return;
Tommi1050fbc2021-06-03 15:58:28327
Sebastian Jansson8e0b15b2018-04-18 17:19:22328 TRACE_EVENT_INSTANT0("webrtc", "VideoSendStream::Start");
Stefan Holmer9416ef82018-07-19 08:34:38329 rtp_video_sender_->SetActive(true);
Sebastian Jansson8e0b15b2018-04-18 17:19:22330 StartupVideoSendStream();
331}
332
333void VideoSendStreamImpl::StartupVideoSendStream() {
Tommifa3ce632021-06-03 10:06:02334 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Tommi1050fbc2021-06-03 15:58:28335
336 transport_queue_safety_->SetAlive();
337
Sebastian Jansson464a5572019-02-12 12:32:32338 bitrate_allocator_->AddObserver(this, GetAllocationConfig());
Sebastian Jansson8e0b15b2018-04-18 17:19:22339 // Start monitoring encoder activity.
340 {
Sebastian Janssonecb68972019-01-18 09:30:54341 RTC_DCHECK(!check_encoder_activity_task_.Running());
342
343 activity_ = false;
344 timed_out_ = false;
Sebastian Janssoncda86dd2019-03-11 16:26:36345 check_encoder_activity_task_ = RepeatingTaskHandle::DelayedStart(
Tommifa3ce632021-06-03 10:06:02346 rtp_transport_queue_->Get(), kEncoderTimeOut, [this] {
347 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Sebastian Janssonecb68972019-01-18 09:30:54348 if (!activity_) {
349 if (!timed_out_) {
350 SignalEncoderTimedOut();
351 }
352 timed_out_ = true;
Ilya Nikolaevskiyaa9aa572019-04-11 07:20:24353 disable_padding_ = true;
Sebastian Janssonecb68972019-01-18 09:30:54354 } else if (timed_out_) {
355 SignalEncoderActive();
356 timed_out_ = false;
357 }
358 activity_ = false;
359 return kEncoderTimeOut;
360 });
Sebastian Jansson8e0b15b2018-04-18 17:19:22361 }
362
363 video_stream_encoder_->SendKeyFrame();
364}
365
366void VideoSendStreamImpl::Stop() {
Tommifa3ce632021-06-03 10:06:02367 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Evan Shrubsole0db33962020-11-25 13:54:58368 RTC_LOG(LS_INFO) << "VideoSendStreamImpl::Stop";
Stefan Holmer9416ef82018-07-19 08:34:38369 if (!rtp_video_sender_->IsActive())
Sebastian Jansson8e0b15b2018-04-18 17:19:22370 return;
Tommi1050fbc2021-06-03 15:58:28371
372 RTC_DCHECK(transport_queue_safety_->alive());
Sebastian Jansson8e0b15b2018-04-18 17:19:22373 TRACE_EVENT_INSTANT0("webrtc", "VideoSendStream::Stop");
Stefan Holmer9416ef82018-07-19 08:34:38374 rtp_video_sender_->SetActive(false);
Sebastian Jansson8e0b15b2018-04-18 17:19:22375 StopVideoSendStream();
376}
377
Tommi1050fbc2021-06-03 15:58:28378// RTC_RUN_ON(rtp_transport_queue_)
Sebastian Jansson8e0b15b2018-04-18 17:19:22379void VideoSendStreamImpl::StopVideoSendStream() {
380 bitrate_allocator_->RemoveObserver(this);
Sebastian Janssonecb68972019-01-18 09:30:54381 check_encoder_activity_task_.Stop();
Florent Castellia8336d32019-09-09 11:36:55382 video_stream_encoder_->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
Ying Wang9b881ab2020-02-07 13:29:32383 DataRate::Zero(), 0, 0, 0);
Sebastian Jansson8e0b15b2018-04-18 17:19:22384 stats_proxy_->OnSetEncoderTargetRate(0);
Tommi1050fbc2021-06-03 15:58:28385 transport_queue_safety_->SetNotAlive();
Sebastian Jansson8e0b15b2018-04-18 17:19:22386}
387
388void VideoSendStreamImpl::SignalEncoderTimedOut() {
Tommifa3ce632021-06-03 10:06:02389 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Sebastian Janssonecb68972019-01-18 09:30:54390 // If the encoder has not produced anything the last kEncoderTimeOut and it
Sebastian Jansson8e0b15b2018-04-18 17:19:22391 // is supposed to, deregister as BitrateAllocatorObserver. This can happen
392 // if a camera stops producing frames.
393 if (encoder_target_rate_bps_ > 0) {
394 RTC_LOG(LS_INFO) << "SignalEncoderTimedOut, Encoder timed out.";
395 bitrate_allocator_->RemoveObserver(this);
396 }
397}
398
399void VideoSendStreamImpl::OnBitrateAllocationUpdated(
Erik Språng566124a2018-04-23 10:32:22400 const VideoBitrateAllocation& allocation) {
Tommifa3ce632021-06-03 10:06:02401 if (!rtp_transport_queue_->IsCurrent()) {
Tommi1050fbc2021-06-03 15:58:28402 rtp_transport_queue_->PostTask(ToQueuedTask(transport_queue_safety_, [=] {
403 OnBitrateAllocationUpdated(allocation);
404 }));
Erik Språng4e193e42018-09-14 17:01:58405 return;
406 }
407
Tommifa3ce632021-06-03 10:06:02408 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Erik Språng4e193e42018-09-14 17:01:58409
Sebastian Jansson572c60f2019-03-04 17:30:41410 int64_t now_ms = clock_->TimeInMilliseconds();
Erik Språngf4ef2dd2018-09-11 10:37:51411 if (encoder_target_rate_bps_ != 0) {
Erik Språng4e193e42018-09-14 17:01:58412 if (video_bitrate_allocation_context_) {
413 // If new allocation is within kMaxVbaSizeDifferencePercent larger than
414 // the previously sent allocation and the same streams are still enabled,
415 // it is considered "similar". We do not want send similar allocations
416 // more once per kMaxVbaThrottleTimeMs.
417 const VideoBitrateAllocation& last =
418 video_bitrate_allocation_context_->last_sent_allocation;
419 const bool is_similar =
420 allocation.get_sum_bps() >= last.get_sum_bps() &&
421 allocation.get_sum_bps() <
422 (last.get_sum_bps() * (100 + kMaxVbaSizeDifferencePercent)) /
423 100 &&
424 SameStreamsEnabled(allocation, last);
425 if (is_similar &&
426 (now_ms - video_bitrate_allocation_context_->last_send_time_ms) <
427 kMaxVbaThrottleTimeMs) {
428 // This allocation is too similar, cache it and return.
429 video_bitrate_allocation_context_->throttled_allocation = allocation;
430 return;
431 }
432 } else {
433 video_bitrate_allocation_context_.emplace();
434 }
435
436 video_bitrate_allocation_context_->last_sent_allocation = allocation;
437 video_bitrate_allocation_context_->throttled_allocation.reset();
438 video_bitrate_allocation_context_->last_send_time_ms = now_ms;
439
Erik Språngf4ef2dd2018-09-11 10:37:51440 // Send bitrate allocation metadata only if encoder is not paused.
441 rtp_video_sender_->OnBitrateAllocationUpdated(allocation);
442 }
Sebastian Jansson8e0b15b2018-04-18 17:19:22443}
444
Per Kjellandera9434842020-10-15 15:53:22445void VideoSendStreamImpl::OnVideoLayersAllocationUpdated(
446 VideoLayersAllocation allocation) {
Per Kjellanderda06e8f2021-01-07 18:37:00447 // OnVideoLayersAllocationUpdated is handled on the encoder task queue in
448 // order to not race with OnEncodedImage callbacks.
Per Kjellanderaf704182020-10-16 08:30:40449 rtp_video_sender_->OnVideoLayersAllocationUpdated(allocation);
Per Kjellandera9434842020-10-15 15:53:22450}
451
Sebastian Jansson8e0b15b2018-04-18 17:19:22452void VideoSendStreamImpl::SignalEncoderActive() {
Tommifa3ce632021-06-03 10:06:02453 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Ilya Nikolaevskiyaa9aa572019-04-11 07:20:24454 if (rtp_video_sender_->IsActive()) {
455 RTC_LOG(LS_INFO) << "SignalEncoderActive, Encoder is active.";
456 bitrate_allocator_->AddObserver(this, GetAllocationConfig());
457 }
Sebastian Jansson464a5572019-02-12 12:32:32458}
459
460MediaStreamAllocationConfig VideoSendStreamImpl::GetAllocationConfig() const {
461 return MediaStreamAllocationConfig{
462 static_cast<uint32_t>(encoder_min_bitrate_bps_),
463 encoder_max_bitrate_bps_,
Ilya Nikolaevskiyaa9aa572019-04-11 07:20:24464 static_cast<uint32_t>(disable_padding_ ? 0 : max_padding_bitrate_),
Sebastian Jansson464a5572019-02-12 12:32:32465 /* priority_bitrate */ 0,
466 !config_->suspend_below_min_bitrate,
Sebastian Jansson464a5572019-02-12 12:32:32467 encoder_bitrate_priority_};
Sebastian Jansson8e0b15b2018-04-18 17:19:22468}
469
470void VideoSendStreamImpl::OnEncoderConfigurationChanged(
471 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 12:10:27472 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 10:09:46473 VideoEncoderConfig::ContentType content_type,
Sebastian Jansson8e0b15b2018-04-18 17:19:22474 int min_transmit_bitrate_bps) {
Tommifa3ce632021-06-03 10:06:02475 if (!rtp_transport_queue_->IsCurrent()) {
Tommi1050fbc2021-06-03 15:58:28476 rtp_transport_queue_->PostTask(ToQueuedTask(
477 transport_queue_safety_,
478 [this, streams = std::move(streams), is_svc, content_type,
479 min_transmit_bitrate_bps]() mutable {
480 OnEncoderConfigurationChanged(std::move(streams), is_svc,
481 content_type, min_transmit_bitrate_bps);
482 }));
Sebastian Jansson8e0b15b2018-04-18 17:19:22483 return;
484 }
Ilya Nikolaevskiy93be66c2020-04-02 12:10:27485
Sebastian Jansson8e0b15b2018-04-18 17:19:22486 RTC_DCHECK_GE(config_->rtp.ssrcs.size(), streams.size());
487 TRACE_EVENT0("webrtc", "VideoSendStream::OnEncoderConfigurationChanged");
Tommifa3ce632021-06-03 10:06:02488 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Sebastian Jansson8e0b15b2018-04-18 17:19:22489
Elad Alon80f53b72019-10-11 14:19:43490 const VideoCodecType codec_type =
491 PayloadStringToCodecType(config_->rtp.payload_name);
492
493 const absl::optional<DataRate> experimental_min_bitrate =
494 GetExperimentalMinVideoBitrate(codec_type);
Sebastian Jansson8e0b15b2018-04-18 17:19:22495 encoder_min_bitrate_bps_ =
Elad Alonc67a4d62019-10-11 14:54:18496 experimental_min_bitrate
497 ? experimental_min_bitrate->bps()
498 : std::max(streams[0].min_bitrate_bps, kDefaultMinVideoBitrateBps);
499
Sebastian Jansson8e0b15b2018-04-18 17:19:22500 encoder_max_bitrate_bps_ = 0;
501 double stream_bitrate_priority_sum = 0;
502 for (const auto& stream : streams) {
503 // We don't want to allocate more bitrate than needed to inactive streams.
504 encoder_max_bitrate_bps_ += stream.active ? stream.max_bitrate_bps : 0;
505 if (stream.bitrate_priority) {
506 RTC_DCHECK_GT(*stream.bitrate_priority, 0);
507 stream_bitrate_priority_sum += *stream.bitrate_priority;
508 }
509 }
510 RTC_DCHECK_GT(stream_bitrate_priority_sum, 0);
511 encoder_bitrate_priority_ = stream_bitrate_priority_sum;
512 encoder_max_bitrate_bps_ =
513 std::max(static_cast<uint32_t>(encoder_min_bitrate_bps_),
514 encoder_max_bitrate_bps_);
“Michael277a6562018-06-01 19:09:19515
Rasmus Brandtc402dbe2019-02-04 10:09:46516 // TODO(bugs.webrtc.org/10266): Query the VideoBitrateAllocator instead.
Ilya Nikolaevskiy93be66c2020-04-02 12:10:27517 max_padding_bitrate_ = CalculateMaxPadBitrateBps(
518 streams, is_svc, content_type, min_transmit_bitrate_bps,
519 config_->suspend_below_min_bitrate, has_alr_probing_);
Sebastian Jansson8e0b15b2018-04-18 17:19:22520
521 // Clear stats for disabled layers.
522 for (size_t i = streams.size(); i < config_->rtp.ssrcs.size(); ++i) {
523 stats_proxy_->OnInactiveSsrc(config_->rtp.ssrcs[i]);
524 }
525
526 const size_t num_temporal_layers =
527 streams.back().num_temporal_layers.value_or(1);
Stefan Holmer64be7fa2018-10-04 13:21:55528
529 rtp_video_sender_->SetEncodingData(streams[0].width, streams[0].height,
530 num_temporal_layers);
Sebastian Jansson8e0b15b2018-04-18 17:19:22531
Jakob Ivarssonf9d5e552021-06-16 16:03:16532 if (rtp_video_sender_->IsActive()) {
Sebastian Jansson8e0b15b2018-04-18 17:19:22533 // The send stream is started already. Update the allocator with new bitrate
534 // limits.
Sebastian Jansson464a5572019-02-12 12:32:32535 bitrate_allocator_->AddObserver(this, GetAllocationConfig());
Sebastian Jansson8e0b15b2018-04-18 17:19:22536 }
537}
538
539EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage(
540 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 15:30:36541 const CodecSpecificInfo* codec_specific_info) {
Sebastian Jansson8e0b15b2018-04-18 17:19:22542 // Encoded is called on whatever thread the real encoder implementation run
543 // on. In the case of hardware encoders, there might be several encoders
544 // running in parallel on different threads.
Sebastian Janssonecb68972019-01-18 09:30:54545
546 // Indicate that there still is activity going on.
547 activity_ = true;
Sebastian Jansson8e0b15b2018-04-18 17:19:22548
Ilya Nikolaevskiyaa9aa572019-04-11 07:20:24549 auto enable_padding_task = [this]() {
550 if (disable_padding_) {
Tommifa3ce632021-06-03 10:06:02551 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Ilya Nikolaevskiyaa9aa572019-04-11 07:20:24552 disable_padding_ = false;
553 // To ensure that padding bitrate is propagated to the bitrate allocator.
554 SignalEncoderActive();
555 }
556 };
Tommifa3ce632021-06-03 10:06:02557 if (!rtp_transport_queue_->IsCurrent()) {
Tommi1050fbc2021-06-03 15:58:28558 rtp_transport_queue_->PostTask(
559 ToQueuedTask(transport_queue_safety_, std::move(enable_padding_task)));
Ilya Nikolaevskiyaa9aa572019-04-11 07:20:24560 } else {
561 enable_padding_task();
562 }
563
Niels Möller46879152019-01-07 14:54:47564 EncodedImageCallback::Result result(EncodedImageCallback::Result::OK);
Danil Chapovalov2549f172020-08-12 15:30:36565 result =
566 rtp_video_sender_->OnEncodedImage(encoded_image, codec_specific_info);
Erik Språng4e193e42018-09-14 17:01:58567 // Check if there's a throttled VideoBitrateAllocation that we should try
568 // sending.
Tommi1050fbc2021-06-03 15:58:28569 auto update_task = [this]() {
570 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
571 auto& context = video_bitrate_allocation_context_;
572 if (context && context->throttled_allocation) {
573 OnBitrateAllocationUpdated(*context->throttled_allocation);
Erik Språng4e193e42018-09-14 17:01:58574 }
575 };
Tommifa3ce632021-06-03 10:06:02576 if (!rtp_transport_queue_->IsCurrent()) {
Tommi1050fbc2021-06-03 15:58:28577 rtp_transport_queue_->PostTask(
578 ToQueuedTask(transport_queue_safety_, std::move(update_task)));
Erik Språng4e193e42018-09-14 17:01:58579 } else {
580 update_task();
581 }
582
583 return result;
Sebastian Jansson8e0b15b2018-04-18 17:19:22584}
585
Jakob Ivarsson159b4172019-10-30 13:02:14586void VideoSendStreamImpl::OnDroppedFrame(
587 EncodedImageCallback::DropReason reason) {
588 activity_ = true;
589}
590
Sebastian Jansson8e0b15b2018-04-18 17:19:22591std::map<uint32_t, RtpState> VideoSendStreamImpl::GetRtpStates() const {
Stefan Holmer9416ef82018-07-19 08:34:38592 return rtp_video_sender_->GetRtpStates();
Sebastian Jansson8e0b15b2018-04-18 17:19:22593}
594
595std::map<uint32_t, RtpPayloadState> VideoSendStreamImpl::GetRtpPayloadStates()
596 const {
Stefan Holmer9416ef82018-07-19 08:34:38597 return rtp_video_sender_->GetRtpPayloadStates();
Sebastian Jansson8e0b15b2018-04-18 17:19:22598}
599
Sebastian Janssonc0e4d452018-10-25 13:08:32600uint32_t VideoSendStreamImpl::OnBitrateUpdated(BitrateAllocationUpdate update) {
Tommifa3ce632021-06-03 10:06:02601 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Stefan Holmer9416ef82018-07-19 08:34:38602 RTC_DCHECK(rtp_video_sender_->IsActive())
Sebastian Jansson8e0b15b2018-04-18 17:19:22603 << "VideoSendStream::Start has not been called.";
604
Florent Castellia8336d32019-09-09 11:36:55605 // When the BWE algorithm doesn't pass a stable estimate, we'll use the
606 // unstable one instead.
607 if (update.stable_target_bitrate.IsZero()) {
608 update.stable_target_bitrate = update.target_bitrate;
609 }
610
Sebastian Jansson82ed2e82019-10-15 13:58:37611 rtp_video_sender_->OnBitrateUpdated(update, stats_proxy_->GetSendFrameRate());
Stefan Holmer64be7fa2018-10-04 13:21:55612 encoder_target_rate_bps_ = rtp_video_sender_->GetPayloadBitrateBps();
Erik Språng26111642019-03-26 10:09:04613 const uint32_t protection_bitrate_bps =
614 rtp_video_sender_->GetProtectionBitrateBps();
Erik Språng4c6ca302019-04-08 13:14:01615 DataRate link_allocation = DataRate::Zero();
616 if (encoder_target_rate_bps_ > protection_bitrate_bps) {
617 link_allocation =
Danil Chapovalovcad3e0e2020-02-17 17:46:07618 DataRate::BitsPerSec(encoder_target_rate_bps_ - protection_bitrate_bps);
Erik Språng610c7632019-03-06 14:37:33619 }
Florent Castellia8336d32019-09-09 11:36:55620 DataRate overhead =
Danil Chapovalovcad3e0e2020-02-17 17:46:07621 update.target_bitrate - DataRate::BitsPerSec(encoder_target_rate_bps_);
Florent Castellia8336d32019-09-09 11:36:55622 DataRate encoder_stable_target_rate = update.stable_target_bitrate;
623 if (encoder_stable_target_rate > overhead) {
624 encoder_stable_target_rate = encoder_stable_target_rate - overhead;
625 } else {
Danil Chapovalovcad3e0e2020-02-17 17:46:07626 encoder_stable_target_rate = DataRate::BitsPerSec(encoder_target_rate_bps_);
Florent Castellia8336d32019-09-09 11:36:55627 }
628
Sebastian Jansson8e0b15b2018-04-18 17:19:22629 encoder_target_rate_bps_ =
630 std::min(encoder_max_bitrate_bps_, encoder_target_rate_bps_);
Erik Språng4c6ca302019-04-08 13:14:01631
Danil Chapovalovcad3e0e2020-02-17 17:46:07632 encoder_stable_target_rate =
633 std::min(DataRate::BitsPerSec(encoder_max_bitrate_bps_),
634 encoder_stable_target_rate);
Florent Castellia8336d32019-09-09 11:36:55635
Danil Chapovalovcad3e0e2020-02-17 17:46:07636 DataRate encoder_target_rate = DataRate::BitsPerSec(encoder_target_rate_bps_);
Erik Språng4c6ca302019-04-08 13:14:01637 link_allocation = std::max(encoder_target_rate, link_allocation);
Sebastian Jansson13e59032018-11-21 18:13:07638 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 11:36:55639 encoder_target_rate, encoder_stable_target_rate, link_allocation,
Sebastian Jansson13e59032018-11-21 18:13:07640 rtc::dchecked_cast<uint8_t>(update.packet_loss_ratio * 256),
Ying Wang9b881ab2020-02-07 13:29:32641 update.round_trip_time.ms(), update.cwnd_reduce_ratio);
Sebastian Jansson8e0b15b2018-04-18 17:19:22642 stats_proxy_->OnSetEncoderTargetRate(encoder_target_rate_bps_);
Erik Språng26111642019-03-26 10:09:04643 return protection_bitrate_bps;
Sebastian Jansson8e0b15b2018-04-18 17:19:22644}
645
Sebastian Jansson8e0b15b2018-04-18 17:19:22646} // namespace internal
647} // namespace webrtc