blob: 0d2a03d27bf8022eb072e52863bf399f9288cdbb [file] [log] [blame]
solenbergc7a8b082015-10-16 21:35:071/*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 04:47:3111#include "audio/audio_send_stream.h"
solenbergc7a8b082015-10-16 21:35:0712
Mirko Bonadei317a1f02019-09-17 15:06:1813#include <memory>
solenbergc7a8b082015-10-16 21:35:0714#include <string>
ossu20a4b3f2017-04-27 09:08:5215#include <utility>
16#include <vector>
solenbergc7a8b082015-10-16 21:35:0717
Yves Gerey988cc082018-10-23 10:03:0118#include "api/audio_codecs/audio_encoder.h"
19#include "api/audio_codecs/audio_encoder_factory.h"
20#include "api/audio_codecs/audio_format.h"
21#include "api/call/transport.h"
Steve Anton10542f22019-01-11 17:11:0022#include "api/crypto/frame_encryptor_interface.h"
Artem Titov741daaf2019-03-21 13:37:3623#include "api/function_view.h"
Danil Chapovalov83bbe912019-08-07 10:24:5324#include "api/rtc_event_log/rtc_event_log.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3125#include "audio/audio_state.h"
Yves Gerey988cc082018-10-23 10:03:0126#include "audio/channel_send.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3127#include "audio/conversion.h"
Yves Gerey988cc082018-10-23 10:03:0128#include "call/rtp_config.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3129#include "call/rtp_transport_controller_send_interface.h"
Yves Gerey988cc082018-10-23 10:03:0130#include "common_audio/vad/include/vad.h"
Oskar Sundbom56ef3052018-10-30 15:11:0231#include "logging/rtc_event_log/events/rtc_event_audio_send_stream_config.h"
Oskar Sundbom56ef3052018-10-30 15:11:0232#include "logging/rtc_event_log/rtc_stream_config.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3133#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h"
Philipp Hanckeedcd9662020-06-24 10:52:4234#include "modules/audio_coding/codecs/red/audio_encoder_copy_red.h"
Yves Gerey988cc082018-10-23 10:03:0135#include "modules/audio_processing/include/audio_processing.h"
Sebastian Jansson6298b562020-01-14 16:55:1936#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3137#include "rtc_base/checks.h"
38#include "rtc_base/event.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3139#include "rtc_base/logging.h"
Jonas Olssonabbe8412018-04-03 11:40:0540#include "rtc_base/strings/audio_format_to_string.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3141#include "rtc_base/task_queue.h"
Alex Narestcedd3512017-12-07 19:54:5542#include "system_wrappers/include/field_trial.h"
solenbergc7a8b082015-10-16 21:35:0743
44namespace webrtc {
Fredrik Solenberg8f5787a2018-01-11 12:52:3045namespace {
elad.alond12a8e12017-03-23 18:04:4846
Oskar Sundbom56ef3052018-10-30 15:11:0247void UpdateEventLogStreamConfig(RtcEventLog* event_log,
48 const AudioSendStream::Config& config,
49 const AudioSendStream::Config* old_config) {
50 using SendCodecSpec = AudioSendStream::Config::SendCodecSpec;
51 // Only update if any of the things we log have changed.
52 auto payload_types_equal = [](const absl::optional<SendCodecSpec>& a,
53 const absl::optional<SendCodecSpec>& b) {
54 if (a.has_value() && b.has_value()) {
55 return a->format.name == b->format.name &&
56 a->payload_type == b->payload_type;
57 }
58 return !a.has_value() && !b.has_value();
59 };
60
61 if (old_config && config.rtp.ssrc == old_config->rtp.ssrc &&
62 config.rtp.extensions == old_config->rtp.extensions &&
63 payload_types_equal(config.send_codec_spec,
64 old_config->send_codec_spec)) {
65 return;
66 }
67
Mirko Bonadei317a1f02019-09-17 15:06:1868 auto rtclog_config = std::make_unique<rtclog::StreamConfig>();
Oskar Sundbom56ef3052018-10-30 15:11:0269 rtclog_config->local_ssrc = config.rtp.ssrc;
70 rtclog_config->rtp_extensions = config.rtp.extensions;
71 if (config.send_codec_spec) {
72 rtclog_config->codecs.emplace_back(config.send_codec_spec->format.name,
73 config.send_codec_spec->payload_type, 0);
74 }
Mirko Bonadei317a1f02019-09-17 15:06:1875 event_log->Log(std::make_unique<RtcEventAudioSendStreamConfig>(
Oskar Sundbom56ef3052018-10-30 15:11:0276 std::move(rtclog_config)));
77}
ossu20a4b3f2017-04-27 09:08:5278} // namespace
79
Sebastian Janssonf23131f2019-10-03 08:03:5580constexpr char AudioAllocationConfig::kKey[];
81
82std::unique_ptr<StructParametersParser> AudioAllocationConfig::Parser() {
83 return StructParametersParser::Create( //
84 "min", &min_bitrate, //
85 "max", &max_bitrate, //
86 "prio_rate", &priority_bitrate, //
87 "prio_rate_raw", &priority_bitrate_raw, //
88 "rate_prio", &bitrate_priority);
89}
90
91AudioAllocationConfig::AudioAllocationConfig() {
92 Parser()->Parse(field_trial::FindFullName(kKey));
93 if (priority_bitrate_raw && !priority_bitrate.IsZero()) {
94 RTC_LOG(LS_WARNING) << "'priority_bitrate' and '_raw' are mutually "
95 "exclusive but both were configured.";
96 }
97}
98
99namespace internal {
solenberg566ef242015-11-06 23:34:49100AudioSendStream::AudioSendStream(
Sebastian Jansson977b3352019-03-04 16:43:34101 Clock* clock,
solenberg566ef242015-11-06 23:34:49102 const webrtc::AudioSendStream::Config& config,
Stefan Holmerb86d4e42015-12-07 09:26:18103 const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
Sebastian Jansson44dd9f22019-03-08 13:50:30104 TaskQueueFactory* task_queue_factory,
Niels Möller7d76a312018-10-26 10:57:07105 RtpTransportControllerSendInterface* rtp_transport,
Niels Möller67b011d2018-10-22 11:00:40106 BitrateAllocatorInterface* bitrate_allocator,
michaelt9332b7d2016-11-30 15:51:13107 RtcEventLog* event_log,
ossuc3d4b482017-05-23 13:07:11108 RtcpRttStats* rtcp_rtt_stats,
Sam Zackrissonff058162018-11-20 16:15:13109 const absl::optional<RtpState>& suspended_rtp_state)
Sebastian Jansson977b3352019-03-04 16:43:34110 : AudioSendStream(clock,
111 config,
Fredrik Solenberg8f5787a2018-01-11 12:52:30112 audio_state,
Sebastian Jansson44dd9f22019-03-08 13:50:30113 task_queue_factory,
Niels Möller7d76a312018-10-26 10:57:07114 rtp_transport,
Fredrik Solenberg8f5787a2018-01-11 12:52:30115 bitrate_allocator,
116 event_log,
Fredrik Solenberg8f5787a2018-01-11 12:52:30117 suspended_rtp_state,
Erik Språng2b4d2f32020-06-29 14:37:44118 voe::CreateChannelSend(
119 clock,
120 task_queue_factory,
Erik Språng2b4d2f32020-06-29 14:37:44121 config.send_transport,
122 rtcp_rtt_stats,
123 event_log,
124 config.frame_encryptor,
125 config.crypto_options,
126 config.rtp.extmap_allow_mixed,
127 config.rtcp_report_interval_ms,
128 config.rtp.ssrc,
129 config.frame_transformer,
130 rtp_transport->transport_feedback_observer())) {}
Fredrik Solenberg8f5787a2018-01-11 12:52:30131
132AudioSendStream::AudioSendStream(
Sebastian Jansson977b3352019-03-04 16:43:34133 Clock* clock,
Fredrik Solenberg8f5787a2018-01-11 12:52:30134 const webrtc::AudioSendStream::Config& config,
135 const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
Sebastian Jansson44dd9f22019-03-08 13:50:30136 TaskQueueFactory* task_queue_factory,
Niels Möller7d76a312018-10-26 10:57:07137 RtpTransportControllerSendInterface* rtp_transport,
Niels Möller67b011d2018-10-22 11:00:40138 BitrateAllocatorInterface* bitrate_allocator,
Fredrik Solenberg8f5787a2018-01-11 12:52:30139 RtcEventLog* event_log,
Danil Chapovalovb9b146c2018-06-15 10:28:07140 const absl::optional<RtpState>& suspended_rtp_state,
Niels Möllerdced9f62018-11-19 09:27:07141 std::unique_ptr<voe::ChannelSendInterface> channel_send)
Sebastian Jansson977b3352019-03-04 16:43:34142 : clock_(clock),
Markus Handell3907e7b2021-06-01 07:07:20143 rtp_transport_queue_(rtp_transport->GetWorkerQueue()),
Sebastian Janssonf23131f2019-10-03 08:03:55144 allocate_audio_without_feedback_(
145 field_trial::IsEnabled("WebRTC-Audio-ABWENoTWCC")),
146 enable_audio_alr_probing_(
147 !field_trial::IsDisabled("WebRTC-Audio-AlrProbing")),
148 send_side_bwe_with_overhead_(
Jakob Ivarsson36274f92020-10-22 11:01:07149 !field_trial::IsDisabled("WebRTC-SendSideBwe-WithOverhead")),
Bjorn A Mellem7a9a0922019-11-26 17:19:40150 config_(Config(/*send_transport=*/nullptr)),
mflodman86cc6ff2016-07-26 11:44:06151 audio_state_(audio_state),
Niels Möllerdced9f62018-11-19 09:27:07152 channel_send_(std::move(channel_send)),
ossu20a4b3f2017-04-27 09:08:52153 event_log_(event_log),
Sebastian Jansson62aee932019-10-02 10:27:06154 use_legacy_overhead_calculation_(
Sebastian Janssonbef818d2020-01-30 13:09:48155 field_trial::IsEnabled("WebRTC-Audio-LegacyOverhead")),
michaeltf4caaab2017-01-17 07:55:07156 bitrate_allocator_(bitrate_allocator),
Niels Möller7d76a312018-10-26 10:57:07157 rtp_transport_(rtp_transport),
Sebastian Jansson6298b562020-01-14 16:55:19158 rtp_rtcp_module_(channel_send_->GetRtpRtcp()),
Sam Zackrissonff058162018-11-20 16:15:13159 suspended_rtp_state_(suspended_rtp_state) {
Jonas Olsson24ea8222018-01-25 09:14:29160 RTC_LOG(LS_INFO) << "AudioSendStream: " << config.rtp.ssrc;
Markus Handell3907e7b2021-06-01 07:07:20161 RTC_DCHECK(rtp_transport_queue_);
Fredrik Solenberg8f5787a2018-01-11 12:52:30162 RTC_DCHECK(audio_state_);
Niels Möllerdced9f62018-11-19 09:27:07163 RTC_DCHECK(channel_send_);
Fredrik Solenberg8f5787a2018-01-11 12:52:30164 RTC_DCHECK(bitrate_allocator_);
Sebastian Jansson0b698262019-03-07 08:17:19165 RTC_DCHECK(rtp_transport);
166
ossuc3d4b482017-05-23 13:07:11167 RTC_DCHECK(rtp_rtcp_module_);
mflodman3d7db262016-04-29 07:57:13168
Artem Titova2088612021-02-03 12:33:28169 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Sebastian Jansson35cf9e72019-10-04 07:30:32170 ConfigureStream(config, true);
Artem Titova2088612021-02-03 12:33:28171 UpdateCachedTargetAudioBitrateConstraints();
Sebastian Janssonc01367d2019-04-08 13:20:44172 pacer_thread_checker_.Detach();
solenbergc7a8b082015-10-16 21:35:07173}
174
175AudioSendStream::~AudioSendStream() {
Artem Titova2088612021-02-03 12:33:28176 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Jonas Olsson24ea8222018-01-25 09:14:29177 RTC_LOG(LS_INFO) << "~AudioSendStream: " << config_.rtp.ssrc;
Fredrik Solenberg2a877972017-12-15 15:42:15178 RTC_DCHECK(!sending_);
Sebastian Jansson0a6510d2019-10-04 07:31:08179 channel_send_->ResetSenderCongestionControlObjects();
Sebastian Jansson8672cac2019-03-01 14:57:55180 // Blocking call to synchronize state with worker queue to ensure that there
181 // are no pending tasks left that keeps references to audio.
182 rtc::Event thread_sync_event;
Markus Handell3907e7b2021-06-01 07:07:20183 rtp_transport_queue_->PostTask([&] { thread_sync_event.Set(); });
Sebastian Jansson8672cac2019-03-01 14:57:55184 thread_sync_event.Wait(rtc::Event::kForever);
solenbergc7a8b082015-10-16 21:35:07185}
186
eladalonabbc4302017-07-26 09:09:44187const webrtc::AudioSendStream::Config& AudioSendStream::GetConfig() const {
Artem Titova2088612021-02-03 12:33:28188 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
eladalonabbc4302017-07-26 09:09:44189 return config_;
190}
191
ossu20a4b3f2017-04-27 09:08:52192void AudioSendStream::Reconfigure(
193 const webrtc::AudioSendStream::Config& new_config) {
Artem Titova2088612021-02-03 12:33:28194 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Sebastian Jansson35cf9e72019-10-04 07:30:32195 ConfigureStream(new_config, false);
ossu20a4b3f2017-04-27 09:08:52196}
197
Alex Narestcedd3512017-12-07 19:54:55198AudioSendStream::ExtensionIds AudioSendStream::FindExtensionIds(
199 const std::vector<RtpExtension>& extensions) {
200 ExtensionIds ids;
201 for (const auto& extension : extensions) {
202 if (extension.uri == RtpExtension::kAudioLevelUri) {
203 ids.audio_level = extension.id;
Sebastian Jansson71c6b562019-08-14 09:31:02204 } else if (extension.uri == RtpExtension::kAbsSendTimeUri) {
205 ids.abs_send_time = extension.id;
Alex Narestcedd3512017-12-07 19:54:55206 } else if (extension.uri == RtpExtension::kTransportSequenceNumberUri) {
207 ids.transport_sequence_number = extension.id;
Steve Antonbb50ce52018-03-26 17:24:32208 } else if (extension.uri == RtpExtension::kMidUri) {
209 ids.mid = extension.id;
Amit Hilbuch77938e62018-12-21 17:23:38210 } else if (extension.uri == RtpExtension::kRidUri) {
211 ids.rid = extension.id;
212 } else if (extension.uri == RtpExtension::kRepairedRidUri) {
213 ids.repaired_rid = extension.id;
Minyue Li74dadc12020-03-05 10:33:13214 } else if (extension.uri == RtpExtension::kAbsoluteCaptureTimeUri) {
215 ids.abs_capture_time = extension.id;
Alex Narestcedd3512017-12-07 19:54:55216 }
217 }
218 return ids;
219}
220
Sebastian Jansson470a5ea2019-01-23 11:37:49221int AudioSendStream::TransportSeqNumId(const AudioSendStream::Config& config) {
222 return FindExtensionIds(config.rtp.extensions).transport_sequence_number;
223}
224
ossu20a4b3f2017-04-27 09:08:52225void AudioSendStream::ConfigureStream(
ossu20a4b3f2017-04-27 09:08:52226 const webrtc::AudioSendStream::Config& new_config,
227 bool first_time) {
Jonas Olsson24ea8222018-01-25 09:14:29228 RTC_LOG(LS_INFO) << "AudioSendStream::ConfigureStream: "
229 << new_config.ToString();
Sebastian Jansson35cf9e72019-10-04 07:30:32230 UpdateEventLogStreamConfig(event_log_, new_config,
231 first_time ? nullptr : &config_);
Oskar Sundbom56ef3052018-10-30 15:11:02232
Sebastian Jansson35cf9e72019-10-04 07:30:32233 const auto& old_config = config_;
ossu20a4b3f2017-04-27 09:08:52234
Niels Möllere9771992018-11-26 09:55:07235 // Configuration parameters which cannot be changed.
236 RTC_DCHECK(first_time ||
237 old_config.send_transport == new_config.send_transport);
Erik Språng70efdde2019-08-21 11:36:20238 RTC_DCHECK(first_time || old_config.rtp.ssrc == new_config.rtp.ssrc);
Sebastian Jansson35cf9e72019-10-04 07:30:32239 if (suspended_rtp_state_ && first_time) {
240 rtp_rtcp_module_->SetRtpState(*suspended_rtp_state_);
ossu20a4b3f2017-04-27 09:08:52241 }
242 if (first_time || old_config.rtp.c_name != new_config.rtp.c_name) {
Sebastian Jansson35cf9e72019-10-04 07:30:32243 channel_send_->SetRTCP_CNAME(new_config.rtp.c_name);
ossu20a4b3f2017-04-27 09:08:52244 }
ossu20a4b3f2017-04-27 09:08:52245
Benjamin Wright84583f62018-10-04 21:22:34246 // Enable the frame encryptor if a new frame encryptor has been provided.
247 if (first_time || new_config.frame_encryptor != old_config.frame_encryptor) {
Sebastian Jansson35cf9e72019-10-04 07:30:32248 channel_send_->SetFrameEncryptor(new_config.frame_encryptor);
Benjamin Wright84583f62018-10-04 21:22:34249 }
250
Johannes Kron9190b822018-10-29 10:22:05251 if (first_time ||
Marina Ciocead2aa8f92020-03-31 09:29:56252 new_config.frame_transformer != old_config.frame_transformer) {
253 channel_send_->SetEncoderToPacketizerFrameTransformer(
254 new_config.frame_transformer);
255 }
256
257 if (first_time ||
Johannes Kron9190b822018-10-29 10:22:05258 new_config.rtp.extmap_allow_mixed != old_config.rtp.extmap_allow_mixed) {
Sebastian Jansson6298b562020-01-14 16:55:19259 rtp_rtcp_module_->SetExtmapAllowMixed(new_config.rtp.extmap_allow_mixed);
Johannes Kron9190b822018-10-29 10:22:05260 }
261
Alex Narestcedd3512017-12-07 19:54:55262 const ExtensionIds old_ids = FindExtensionIds(old_config.rtp.extensions);
263 const ExtensionIds new_ids = FindExtensionIds(new_config.rtp.extensions);
Yves Gerey17048012019-07-26 15:49:52264
ossu20a4b3f2017-04-27 09:08:52265 // Audio level indication
266 if (first_time || new_ids.audio_level != old_ids.audio_level) {
Sebastian Jansson35cf9e72019-10-04 07:30:32267 channel_send_->SetSendAudioLevelIndicationStatus(new_ids.audio_level != 0,
268 new_ids.audio_level);
ossu20a4b3f2017-04-27 09:08:52269 }
Sebastian Jansson71c6b562019-08-14 09:31:02270
271 if (first_time || new_ids.abs_send_time != old_ids.abs_send_time) {
Sebastian Jansson6298b562020-01-14 16:55:19272 rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(
Sebastian Jansson71c6b562019-08-14 09:31:02273 kRtpExtensionAbsoluteSendTime);
274 if (new_ids.abs_send_time) {
Danil Chapovalovd0321c52021-09-14 10:58:51275 rtp_rtcp_module_->RegisterRtpHeaderExtension(AbsoluteSendTime::Uri(),
Sebastian Janssonf39c8152019-10-14 15:32:21276 new_ids.abs_send_time);
Sebastian Jansson71c6b562019-08-14 09:31:02277 }
278 }
279
Sebastian Jansson8d9c5402017-11-15 16:22:16280 bool transport_seq_num_id_changed =
281 new_ids.transport_sequence_number != old_ids.transport_sequence_number;
Sebastian Jansson35cf9e72019-10-04 07:30:32282 if (first_time ||
283 (transport_seq_num_id_changed && !allocate_audio_without_feedback_)) {
ossu1129df22017-06-30 08:38:56284 if (!first_time) {
Sebastian Jansson35cf9e72019-10-04 07:30:32285 channel_send_->ResetSenderCongestionControlObjects();
ossu20a4b3f2017-04-27 09:08:52286 }
287
Sebastian Jansson8d9c5402017-11-15 16:22:16288 RtcpBandwidthObserver* bandwidth_observer = nullptr;
Sebastian Jansson470a5ea2019-01-23 11:37:49289
Jakob Ivarsson47a03e82020-11-23 14:05:44290 if (!allocate_audio_without_feedback_ &&
Sebastian Janssonf23131f2019-10-03 08:03:55291 new_ids.transport_sequence_number != 0) {
Sebastian Jansson6298b562020-01-14 16:55:19292 rtp_rtcp_module_->RegisterRtpHeaderExtension(
Danil Chapovalovd0321c52021-09-14 10:58:51293 TransportSequenceNumber::Uri(), new_ids.transport_sequence_number);
Sebastian Jansson8d9c5402017-11-15 16:22:16294 // Probing in application limited region is only used in combination with
295 // send side congestion control, wich depends on feedback packets which
296 // requires transport sequence numbers to be enabled.
Sebastian Jansson0a6510d2019-10-04 07:31:08297 // Optionally request ALR probing but do not override any existing
298 // request from other streams.
299 if (enable_audio_alr_probing_) {
300 rtp_transport_->EnablePeriodicAlrProbing(true);
Niels Möller7d76a312018-10-26 10:57:07301 }
Sebastian Jansson0a6510d2019-10-04 07:31:08302 bandwidth_observer = rtp_transport_->GetBandwidthObserver();
ossu20a4b3f2017-04-27 09:08:52303 }
Sebastian Jansson0a6510d2019-10-04 07:31:08304 channel_send_->RegisterSenderCongestionControlObjects(rtp_transport_,
305 bandwidth_observer);
ossu20a4b3f2017-04-27 09:08:52306 }
Steve Antonbb50ce52018-03-26 17:24:32307 // MID RTP header extension.
Steve Anton003930a2018-03-29 19:37:21308 if ((first_time || new_ids.mid != old_ids.mid ||
309 new_config.rtp.mid != old_config.rtp.mid) &&
310 new_ids.mid != 0 && !new_config.rtp.mid.empty()) {
Danil Chapovalovd0321c52021-09-14 10:58:51311 rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpMid::Uri(), new_ids.mid);
Sebastian Jansson6298b562020-01-14 16:55:19312 rtp_rtcp_module_->SetMid(new_config.rtp.mid);
Steve Antonbb50ce52018-03-26 17:24:32313 }
314
Amit Hilbuch77938e62018-12-21 17:23:38315 // RID RTP header extension
316 if ((first_time || new_ids.rid != old_ids.rid ||
317 new_ids.repaired_rid != old_ids.repaired_rid ||
318 new_config.rtp.rid != old_config.rtp.rid)) {
Sebastian Jansson6298b562020-01-14 16:55:19319 if (new_ids.rid != 0 || new_ids.repaired_rid != 0) {
320 if (new_config.rtp.rid.empty()) {
Danil Chapovalovd0321c52021-09-14 10:58:51321 rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(RtpStreamId::Uri());
Sebastian Jansson6298b562020-01-14 16:55:19322 } else if (new_ids.repaired_rid != 0) {
Danil Chapovalovd0321c52021-09-14 10:58:51323 rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpStreamId::Uri(),
Sebastian Jansson6298b562020-01-14 16:55:19324 new_ids.repaired_rid);
325 } else {
Danil Chapovalovd0321c52021-09-14 10:58:51326 rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpStreamId::Uri(),
Sebastian Jansson6298b562020-01-14 16:55:19327 new_ids.rid);
328 }
329 }
330 rtp_rtcp_module_->SetRid(new_config.rtp.rid);
Amit Hilbuch77938e62018-12-21 17:23:38331 }
332
Minyue Li74dadc12020-03-05 10:33:13333 if (first_time || new_ids.abs_capture_time != old_ids.abs_capture_time) {
334 rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(
335 kRtpExtensionAbsoluteCaptureTime);
336 if (new_ids.abs_capture_time) {
337 rtp_rtcp_module_->RegisterRtpHeaderExtension(
Danil Chapovalovd0321c52021-09-14 10:58:51338 AbsoluteCaptureTimeExtension::Uri(), new_ids.abs_capture_time);
Minyue Li74dadc12020-03-05 10:33:13339 }
340 }
341
Sebastian Jansson35cf9e72019-10-04 07:30:32342 if (!ReconfigureSendCodec(new_config)) {
Mirko Bonadei675513b2017-11-09 10:09:25343 RTC_LOG(LS_ERROR) << "Failed to set up send codec state.";
ossu20a4b3f2017-04-27 09:08:52344 }
345
Erik Språng04e1bab2020-05-07 16:18:32346 // Set currently known overhead (used in ANA, opus only).
347 {
Markus Handell62872802020-07-06 13:15:07348 MutexLock lock(&overhead_per_packet_lock_);
Erik Språng04e1bab2020-05-07 16:18:32349 UpdateOverheadForEncoder();
350 }
351
Jakob Ivarssond14525e2020-03-06 08:49:29352 channel_send_->CallEncoder([this](AudioEncoder* encoder) {
Artem Titova2088612021-02-03 12:33:28353 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Jakob Ivarssond14525e2020-03-06 08:49:29354 if (!encoder) {
355 return;
356 }
Artem Titova2088612021-02-03 12:33:28357 frame_length_range_ = encoder->GetFrameLengthRange();
358 UpdateCachedTargetAudioBitrateConstraints();
Jakob Ivarssond14525e2020-03-06 08:49:29359 });
360
Sebastian Jansson35cf9e72019-10-04 07:30:32361 if (sending_) {
362 ReconfigureBitrateObserver(new_config);
Oskar Sundbomf85e31b2017-12-20 15:38:09363 }
Artem Titova2088612021-02-03 12:33:28364
Sebastian Jansson35cf9e72019-10-04 07:30:32365 config_ = new_config;
Artem Titova2088612021-02-03 12:33:28366 if (!first_time) {
367 UpdateCachedTargetAudioBitrateConstraints();
368 }
ossu20a4b3f2017-04-27 09:08:52369}
370
solenberg3a941542015-11-16 15:34:50371void AudioSendStream::Start() {
Sebastian Jansson8672cac2019-03-01 14:57:55372 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Fredrik Solenberg2a877972017-12-15 15:42:15373 if (sending_) {
374 return;
375 }
Sebastian Janssonf23131f2019-10-03 08:03:55376 if (!config_.has_dscp && config_.min_bitrate_bps != -1 &&
377 config_.max_bitrate_bps != -1 &&
Sebastian Janssoncd0eedb2019-10-10 11:52:26378 (allocate_audio_without_feedback_ || TransportSeqNumId(config_) != 0)) {
Erik Språngaa59eca2019-07-24 12:52:55379 rtp_transport_->AccountForAudioPacketsInPacedSender(true);
Sebastian Janssonc3eb9fd2020-01-29 16:42:52380 if (send_side_bwe_with_overhead_)
381 rtp_transport_->IncludeOverheadInPacedSender();
Sebastian Janssonb6863962018-10-10 08:23:13382 rtp_rtcp_module_->SetAsPartOfAllocation(true);
Artem Titova2088612021-02-03 12:33:28383 ConfigureBitrateObserver();
Sebastian Janssonb6863962018-10-10 08:23:13384 } else {
385 rtp_rtcp_module_->SetAsPartOfAllocation(false);
mflodman86cc6ff2016-07-26 11:44:06386 }
Niels Möllerdced9f62018-11-19 09:27:07387 channel_send_->StartSend();
Fredrik Solenberg2a877972017-12-15 15:42:15388 sending_ = true;
389 audio_state()->AddSendingStream(this, encoder_sample_rate_hz_,
390 encoder_num_channels_);
solenberg3a941542015-11-16 15:34:50391}
392
393void AudioSendStream::Stop() {
Artem Titova2088612021-02-03 12:33:28394 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Fredrik Solenberg2a877972017-12-15 15:42:15395 if (!sending_) {
396 return;
397 }
398
ossu20a4b3f2017-04-27 09:08:52399 RemoveBitrateObserver();
Niels Möllerdced9f62018-11-19 09:27:07400 channel_send_->StopSend();
Fredrik Solenberg2a877972017-12-15 15:42:15401 sending_ = false;
402 audio_state()->RemoveSendingStream(this);
403}
404
405void AudioSendStream::SendAudioData(std::unique_ptr<AudioFrame> audio_frame) {
406 RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_);
Henrik Boströmd2c336f2019-07-03 15:11:10407 RTC_DCHECK_GT(audio_frame->sample_rate_hz_, 0);
408 double duration = static_cast<double>(audio_frame->samples_per_channel_) /
409 audio_frame->sample_rate_hz_;
410 {
411 // Note: SendAudioData() passes the frame further down the pipeline and it
412 // may eventually get sent. But this method is invoked even if we are not
413 // connected, as long as we have an AudioSendStream (created as a result of
414 // an O/A exchange). This means that we are calculating audio levels whether
415 // or not we are sending samples.
416 // TODO(https://crbug.com/webrtc/10771): All "media-source" related stats
417 // should move from send-streams to the local audio sources or tracks; a
418 // send-stream should not be required to read the microphone audio levels.
Markus Handell62872802020-07-06 13:15:07419 MutexLock lock(&audio_level_lock_);
Henrik Boströmd2c336f2019-07-03 15:11:10420 audio_level_.ComputeLevel(*audio_frame, duration);
421 }
Niels Möllerdced9f62018-11-19 09:27:07422 channel_send_->ProcessAndEncodeAudio(std::move(audio_frame));
solenberg3a941542015-11-16 15:34:50423}
424
solenbergffbbcac2016-11-17 13:25:37425bool AudioSendStream::SendTelephoneEvent(int payload_type,
Yves Gerey665174f2018-06-19 13:03:05426 int payload_frequency,
427 int event,
solenberg8842c3e2016-03-11 11:06:41428 int duration_ms) {
Artem Titova2088612021-02-03 12:33:28429 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller8fb1a6a2019-03-05 13:29:42430 channel_send_->SetSendTelephoneEventPayloadType(payload_type,
431 payload_frequency);
432 return channel_send_->SendTelephoneEventOutband(event, duration_ms);
Fredrik Solenbergb5727682015-12-04 14:22:19433}
434
solenberg94218532016-06-16 17:53:22435void AudioSendStream::SetMuted(bool muted) {
Artem Titova2088612021-02-03 12:33:28436 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möllerdced9f62018-11-19 09:27:07437 channel_send_->SetInputMute(muted);
solenberg94218532016-06-16 17:53:22438}
439
solenbergc7a8b082015-10-16 21:35:07440webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const {
Ivo Creusen56d460902017-11-24 16:29:59441 return GetStats(true);
442}
443
444webrtc::AudioSendStream::Stats AudioSendStream::GetStats(
445 bool has_remote_tracks) const {
Artem Titova2088612021-02-03 12:33:28446 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
solenberg85a04962015-10-27 10:35:21447 webrtc::AudioSendStream::Stats stats;
448 stats.local_ssrc = config_.rtp.ssrc;
Niels Möllerdced9f62018-11-19 09:27:07449 stats.target_bitrate_bps = channel_send_->GetBitrate();
solenberg85a04962015-10-27 10:35:21450
Niels Möllerdced9f62018-11-19 09:27:07451 webrtc::CallSendStatistics call_stats = channel_send_->GetRTCPStatistics();
Niels Möllerac0a4cb2019-10-09 13:01:33452 stats.payload_bytes_sent = call_stats.payload_bytes_sent;
453 stats.header_and_padding_bytes_sent =
454 call_stats.header_and_padding_bytes_sent;
Henrik Boströmcf96e0f2019-04-17 11:51:53455 stats.retransmitted_bytes_sent = call_stats.retransmitted_bytes_sent;
solenberg85a04962015-10-27 10:35:21456 stats.packets_sent = call_stats.packetsSent;
Henrik Boströmcf96e0f2019-04-17 11:51:53457 stats.retransmitted_packets_sent = call_stats.retransmitted_packets_sent;
solenberg8b85de22015-11-16 17:48:04458 // RTT isn't known until a RTCP report is received. Until then, VoiceEngine
459 // returns 0 to indicate an error value.
460 if (call_stats.rttMs > 0) {
461 stats.rtt_ms = call_stats.rttMs;
462 }
ossu20a4b3f2017-04-27 09:08:52463 if (config_.send_codec_spec) {
464 const auto& spec = *config_.send_codec_spec;
465 stats.codec_name = spec.format.name;
Oskar Sundbom2707fb22017-11-16 09:57:35466 stats.codec_payload_type = spec.payload_type;
solenberg85a04962015-10-27 10:35:21467
468 // Get data from the last remote RTCP report.
Niels Möllerdced9f62018-11-19 09:27:07469 for (const auto& block : channel_send_->GetRemoteRTCPReportBlocks()) {
solenberg8b85de22015-11-16 17:48:04470 // Lookup report for send ssrc only.
471 if (block.source_SSRC == stats.local_ssrc) {
472 stats.packets_lost = block.cumulative_num_packets_lost;
473 stats.fraction_lost = Q8ToFloat(block.fraction_lost);
ossu20a4b3f2017-04-27 09:08:52474 // Convert timestamps to milliseconds.
475 if (spec.format.clockrate_hz / 1000 > 0) {
solenberg8b85de22015-11-16 17:48:04476 stats.jitter_ms =
ossu20a4b3f2017-04-27 09:08:52477 block.interarrival_jitter / (spec.format.clockrate_hz / 1000);
solenberg85a04962015-10-27 10:35:21478 }
solenberg8b85de22015-11-16 17:48:04479 break;
solenberg85a04962015-10-27 10:35:21480 }
481 }
482 }
483
Henrik Boströmd2c336f2019-07-03 15:11:10484 {
Markus Handell62872802020-07-06 13:15:07485 MutexLock lock(&audio_level_lock_);
Henrik Boströmd2c336f2019-07-03 15:11:10486 stats.audio_level = audio_level_.LevelFullRange();
487 stats.total_input_energy = audio_level_.TotalEnergy();
488 stats.total_input_duration = audio_level_.TotalDuration();
489 }
solenberg796b8f92017-03-02 01:02:23490
Fredrik Solenberg2a877972017-12-15 15:42:15491 stats.typing_noise_detected = audio_state()->typing_noise_detected();
Niels Möllerdced9f62018-11-19 09:27:07492 stats.ana_statistics = channel_send_->GetANAStatistics();
Per Åhgrencc73ed32020-04-26 21:56:17493
494 AudioProcessing* ap = audio_state_->audio_processing();
495 if (ap) {
496 stats.apm_statistics = ap->GetStatistics(has_remote_tracks);
497 }
solenberg85a04962015-10-27 10:35:21498
Henrik Boström6e436d12019-05-27 10:19:33499 stats.report_block_datas = std::move(call_stats.report_block_datas);
500
Jakob Ivarssone91c9922021-07-06 07:55:43501 stats.nacks_rcvd = call_stats.nacks_rcvd;
502
solenberg85a04962015-10-27 10:35:21503 return stats;
504}
505
Niels Möller8fb1a6a2019-03-05 13:29:42506void AudioSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
Erik Språng2b4d2f32020-06-29 14:37:44507 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller8fb1a6a2019-03-05 13:29:42508 channel_send_->ReceivedRTCPPacket(packet, length);
Artem Titova2088612021-02-03 12:33:28509
510 {
Erik Språng04e1bab2020-05-07 16:18:32511 // Poll if overhead has changed, which it can do if ack triggers us to stop
512 // sending mid/rid.
Markus Handell62872802020-07-06 13:15:07513 MutexLock lock(&overhead_per_packet_lock_);
Erik Språng04e1bab2020-05-07 16:18:32514 UpdateOverheadForEncoder();
Artem Titova2088612021-02-03 12:33:28515 }
516 UpdateCachedTargetAudioBitrateConstraints();
pbos1ba8d392016-05-02 03:18:34517}
518
Sebastian Janssonc0e4d452018-10-25 13:08:32519uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) {
Markus Handell3907e7b2021-06-01 07:07:20520 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Erik Språng04e1bab2020-05-07 16:18:32521
Daniel Lee93562522019-05-03 12:40:13522 // Pick a target bitrate between the constraints. Overrules the allocator if
523 // it 1) allocated a bitrate of zero to disable the stream or 2) allocated a
524 // higher than max to allow for e.g. extra FEC.
Artem Titova2088612021-02-03 12:33:28525 RTC_DCHECK(cached_constraints_.has_value());
526 update.target_bitrate.Clamp(cached_constraints_->min,
527 cached_constraints_->max);
528 update.stable_target_bitrate.Clamp(cached_constraints_->min,
529 cached_constraints_->max);
mflodman86cc6ff2016-07-26 11:44:06530
Sebastian Jansson254d8692018-11-21 18:19:00531 channel_send_->OnBitrateAllocation(update);
mflodman86cc6ff2016-07-26 11:44:06532
533 // The amount of audio protection is not exposed by the encoder, hence
534 // always returning 0.
535 return 0;
536}
537
Anton Sukhanov626015d2019-02-04 23:16:06538void AudioSendStream::SetTransportOverhead(
539 int transport_overhead_per_packet_bytes) {
Artem Titova2088612021-02-03 12:33:28540 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
541 {
542 MutexLock lock(&overhead_per_packet_lock_);
543 transport_overhead_per_packet_bytes_ = transport_overhead_per_packet_bytes;
544 UpdateOverheadForEncoder();
545 }
546 UpdateCachedTargetAudioBitrateConstraints();
Anton Sukhanov626015d2019-02-04 23:16:06547}
548
Anton Sukhanov626015d2019-02-04 23:16:06549void AudioSendStream::UpdateOverheadForEncoder() {
Artem Titova2088612021-02-03 12:33:28550 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Erik Språngcf6544a2020-05-13 12:43:11551 size_t overhead_per_packet_bytes = GetPerPacketOverheadBytes();
552 if (overhead_per_packet_ == overhead_per_packet_bytes) {
553 return;
Bjorn A Mellem413ccc42019-04-26 22:41:05554 }
Erik Språngcf6544a2020-05-13 12:43:11555 overhead_per_packet_ = overhead_per_packet_bytes;
556
Sebastian Jansson14a7cf92019-02-13 14:11:42557 channel_send_->CallEncoder([&](AudioEncoder* encoder) {
558 encoder->OnReceivedOverhead(overhead_per_packet_bytes);
Anton Sukhanov626015d2019-02-04 23:16:06559 });
Artem Titova2088612021-02-03 12:33:28560 if (total_packet_overhead_bytes_ != overhead_per_packet_bytes) {
561 total_packet_overhead_bytes_ = overhead_per_packet_bytes;
562 if (registered_with_allocator_) {
563 ConfigureBitrateObserver();
Sebastian Jansson8672cac2019-03-01 14:57:55564 }
Erik Språng04e1bab2020-05-07 16:18:32565 }
Anton Sukhanov626015d2019-02-04 23:16:06566}
567
568size_t AudioSendStream::TestOnlyGetPerPacketOverheadBytes() const {
Markus Handell62872802020-07-06 13:15:07569 MutexLock lock(&overhead_per_packet_lock_);
Anton Sukhanov626015d2019-02-04 23:16:06570 return GetPerPacketOverheadBytes();
571}
572
573size_t AudioSendStream::GetPerPacketOverheadBytes() const {
574 return transport_overhead_per_packet_bytes_ +
Erik Språng04e1bab2020-05-07 16:18:32575 rtp_rtcp_module_->ExpectedPerPacketOverhead();
michaelt79e05882016-11-08 10:50:09576}
577
ossuc3d4b482017-05-23 13:07:11578RtpState AudioSendStream::GetRtpState() const {
579 return rtp_rtcp_module_->GetRtpState();
580}
581
Niels Möllerdced9f62018-11-19 09:27:07582const voe::ChannelSendInterface* AudioSendStream::GetChannel() const {
583 return channel_send_.get();
Fredrik Solenberg8f5787a2018-01-11 12:52:30584}
585
Fredrik Solenberg2a877972017-12-15 15:42:15586internal::AudioState* AudioSendStream::audio_state() {
587 internal::AudioState* audio_state =
588 static_cast<internal::AudioState*>(audio_state_.get());
589 RTC_DCHECK(audio_state);
590 return audio_state;
591}
592
593const internal::AudioState* AudioSendStream::audio_state() const {
594 internal::AudioState* audio_state =
595 static_cast<internal::AudioState*>(audio_state_.get());
596 RTC_DCHECK(audio_state);
597 return audio_state;
598}
599
Fredrik Solenberg2a877972017-12-15 15:42:15600void AudioSendStream::StoreEncoderProperties(int sample_rate_hz,
601 size_t num_channels) {
Fredrik Solenberg2a877972017-12-15 15:42:15602 encoder_sample_rate_hz_ = sample_rate_hz;
603 encoder_num_channels_ = num_channels;
604 if (sending_) {
605 // Update AudioState's information about the stream.
606 audio_state()->AddSendingStream(this, sample_rate_hz, num_channels);
607 }
608}
609
minyue7a973442016-10-20 10:27:12610// Apply current codec settings to a single voe::Channel used for sending.
Sebastian Jansson35cf9e72019-10-04 07:30:32611bool AudioSendStream::SetupSendCodec(const Config& new_config) {
ossu20a4b3f2017-04-27 09:08:52612 RTC_DCHECK(new_config.send_codec_spec);
613 const auto& spec = *new_config.send_codec_spec;
minyue48368ad2017-05-10 11:06:11614
615 RTC_DCHECK(new_config.encoder_factory);
ossu20a4b3f2017-04-27 09:08:52616 std::unique_ptr<AudioEncoder> encoder =
Karl Wiberg77490b92018-03-21 14:18:42617 new_config.encoder_factory->MakeAudioEncoder(
618 spec.payload_type, spec.format, new_config.codec_pair_id);
minyue7a973442016-10-20 10:27:12619
ossu20a4b3f2017-04-27 09:08:52620 if (!encoder) {
Jonas Olssonabbe8412018-04-03 11:40:05621 RTC_DLOG(LS_ERROR) << "Unable to create encoder for "
622 << rtc::ToString(spec.format);
ossu20a4b3f2017-04-27 09:08:52623 return false;
624 }
Alex Narestbbbe4e12018-07-13 08:32:58625
ossu20a4b3f2017-04-27 09:08:52626 // If a bitrate has been specified for the codec, use it over the
627 // codec's default.
Christoffer Rodbro110c64b2019-03-06 08:51:08628 if (spec.target_bitrate_bps) {
ossu20a4b3f2017-04-27 09:08:52629 encoder->OnReceivedTargetAudioBitrate(*spec.target_bitrate_bps);
minyue7a973442016-10-20 10:27:12630 }
631
ossu20a4b3f2017-04-27 09:08:52632 // Enable ANA if configured (currently only used by Opus).
Mirko Bonadei43564902020-01-29 15:29:36633 if (new_config.audio_network_adaptor_config) {
ossu20a4b3f2017-04-27 09:08:52634 if (encoder->EnableAudioNetworkAdaptor(
Sebastian Jansson35cf9e72019-10-04 07:30:32635 *new_config.audio_network_adaptor_config, event_log_)) {
Jakob Ivarssoned971162020-08-11 12:05:07636 RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC "
637 << new_config.rtp.ssrc;
ossu20a4b3f2017-04-27 09:08:52638 } else {
Jakob Ivarssoned971162020-08-11 12:05:07639 RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC "
640 << new_config.rtp.ssrc;
minyue6b825df2016-10-31 11:08:32641 }
minyue7a973442016-10-20 10:27:12642 }
643
Philipp Hancke1a497562020-05-26 17:12:31644 // Wrap the encoder in an AudioEncoderCNG, if VAD is enabled.
ossu20a4b3f2017-04-27 09:08:52645 if (spec.cng_payload_type) {
Karl Wiberg23659362018-11-01 10:13:44646 AudioEncoderCngConfig cng_config;
ossu20a4b3f2017-04-27 09:08:52647 cng_config.num_channels = encoder->NumChannels();
648 cng_config.payload_type = *spec.cng_payload_type;
649 cng_config.speech_encoder = std::move(encoder);
650 cng_config.vad_mode = Vad::kVadNormal;
Karl Wiberg23659362018-11-01 10:13:44651 encoder = CreateComfortNoiseEncoder(std::move(cng_config));
ossu3b9ff382017-04-27 15:03:42652
Sebastian Jansson35cf9e72019-10-04 07:30:32653 RegisterCngPayloadType(*spec.cng_payload_type,
654 new_config.send_codec_spec->format.clockrate_hz);
minyue7a973442016-10-20 10:27:12655 }
ossu20a4b3f2017-04-27 09:08:52656
Philipp Hanckeedcd9662020-06-24 10:52:42657 // Wrap the encoder in a RED encoder, if RED is enabled.
658 if (spec.red_payload_type) {
659 AudioEncoderCopyRed::Config red_config;
660 red_config.payload_type = *spec.red_payload_type;
661 red_config.speech_encoder = std::move(encoder);
662 encoder = std::make_unique<AudioEncoderCopyRed>(std::move(red_config));
663 }
664
Anton Sukhanov626015d2019-02-04 23:16:06665 // Set currently known overhead (used in ANA, opus only).
666 // If overhead changes later, it will be updated in UpdateOverheadForEncoder.
667 {
Markus Handell62872802020-07-06 13:15:07668 MutexLock lock(&overhead_per_packet_lock_);
Erik Språng04e1bab2020-05-07 16:18:32669 size_t overhead = GetPerPacketOverheadBytes();
670 if (overhead > 0) {
671 encoder->OnReceivedOverhead(overhead);
Bjorn A Mellem413ccc42019-04-26 22:41:05672 }
Anton Sukhanov626015d2019-02-04 23:16:06673 }
674
Sebastian Jansson35cf9e72019-10-04 07:30:32675 StoreEncoderProperties(encoder->SampleRateHz(), encoder->NumChannels());
676 channel_send_->SetEncoder(new_config.send_codec_spec->payload_type,
677 std::move(encoder));
Anton Sukhanov626015d2019-02-04 23:16:06678
minyue7a973442016-10-20 10:27:12679 return true;
680}
681
Sebastian Jansson35cf9e72019-10-04 07:30:32682bool AudioSendStream::ReconfigureSendCodec(const Config& new_config) {
683 const auto& old_config = config_;
minyue-webrtc8de18262017-07-26 12:18:40684
685 if (!new_config.send_codec_spec) {
686 // We cannot de-configure a send codec. So we will do nothing.
687 // By design, the send codec should have not been configured.
688 RTC_DCHECK(!old_config.send_codec_spec);
689 return true;
690 }
691
692 if (new_config.send_codec_spec == old_config.send_codec_spec &&
693 new_config.audio_network_adaptor_config ==
694 old_config.audio_network_adaptor_config) {
ossu20a4b3f2017-04-27 09:08:52695 return true;
696 }
697
698 // If we have no encoder, or the format or payload type's changed, create a
699 // new encoder.
700 if (!old_config.send_codec_spec ||
701 new_config.send_codec_spec->format !=
702 old_config.send_codec_spec->format ||
703 new_config.send_codec_spec->payload_type !=
Philipp Hancke6144b842021-06-04 11:49:27704 old_config.send_codec_spec->payload_type ||
705 new_config.send_codec_spec->red_payload_type !=
706 old_config.send_codec_spec->red_payload_type) {
Sebastian Jansson35cf9e72019-10-04 07:30:32707 return SetupSendCodec(new_config);
ossu20a4b3f2017-04-27 09:08:52708 }
709
Danil Chapovalovb9b146c2018-06-15 10:28:07710 const absl::optional<int>& new_target_bitrate_bps =
ossu20a4b3f2017-04-27 09:08:52711 new_config.send_codec_spec->target_bitrate_bps;
712 // If a bitrate has been specified for the codec, use it over the
713 // codec's default.
Christoffer Rodbro110c64b2019-03-06 08:51:08714 if (new_target_bitrate_bps &&
ossu20a4b3f2017-04-27 09:08:52715 new_target_bitrate_bps !=
716 old_config.send_codec_spec->target_bitrate_bps) {
Sebastian Jansson35cf9e72019-10-04 07:30:32717 channel_send_->CallEncoder([&](AudioEncoder* encoder) {
ossu20a4b3f2017-04-27 09:08:52718 encoder->OnReceivedTargetAudioBitrate(*new_target_bitrate_bps);
719 });
720 }
721
Sebastian Jansson35cf9e72019-10-04 07:30:32722 ReconfigureANA(new_config);
723 ReconfigureCNG(new_config);
ossu20a4b3f2017-04-27 09:08:52724
725 return true;
726}
727
Sebastian Jansson35cf9e72019-10-04 07:30:32728void AudioSendStream::ReconfigureANA(const Config& new_config) {
ossu20a4b3f2017-04-27 09:08:52729 if (new_config.audio_network_adaptor_config ==
Sebastian Jansson35cf9e72019-10-04 07:30:32730 config_.audio_network_adaptor_config) {
ossu20a4b3f2017-04-27 09:08:52731 return;
732 }
Mirko Bonadei43564902020-01-29 15:29:36733 if (new_config.audio_network_adaptor_config) {
Jakob Ivarssonfde2b242020-08-20 14:48:49734 // This lock needs to be acquired before CallEncoder, since it aquires
735 // another lock and we need to maintain the same order at all call sites to
736 // avoid deadlock.
737 MutexLock lock(&overhead_per_packet_lock_);
738 size_t overhead = GetPerPacketOverheadBytes();
Sebastian Jansson35cf9e72019-10-04 07:30:32739 channel_send_->CallEncoder([&](AudioEncoder* encoder) {
ossu20a4b3f2017-04-27 09:08:52740 if (encoder->EnableAudioNetworkAdaptor(
Sebastian Jansson35cf9e72019-10-04 07:30:32741 *new_config.audio_network_adaptor_config, event_log_)) {
Jakob Ivarssoned971162020-08-11 12:05:07742 RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC "
743 << new_config.rtp.ssrc;
Jakob Ivarssonfde2b242020-08-20 14:48:49744 if (overhead > 0) {
745 encoder->OnReceivedOverhead(overhead);
746 }
ossu20a4b3f2017-04-27 09:08:52747 } else {
Jakob Ivarssoned971162020-08-11 12:05:07748 RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC "
749 << new_config.rtp.ssrc;
ossu20a4b3f2017-04-27 09:08:52750 }
751 });
752 } else {
Sebastian Jansson35cf9e72019-10-04 07:30:32753 channel_send_->CallEncoder(
Sebastian Jansson14a7cf92019-02-13 14:11:42754 [&](AudioEncoder* encoder) { encoder->DisableAudioNetworkAdaptor(); });
Jakob Ivarssoned971162020-08-11 12:05:07755 RTC_LOG(LS_INFO) << "Audio network adaptor disabled on SSRC "
756 << new_config.rtp.ssrc;
ossu20a4b3f2017-04-27 09:08:52757 }
758}
759
Sebastian Jansson35cf9e72019-10-04 07:30:32760void AudioSendStream::ReconfigureCNG(const Config& new_config) {
ossu20a4b3f2017-04-27 09:08:52761 if (new_config.send_codec_spec->cng_payload_type ==
Sebastian Jansson35cf9e72019-10-04 07:30:32762 config_.send_codec_spec->cng_payload_type) {
ossu20a4b3f2017-04-27 09:08:52763 return;
764 }
765
ossu3b9ff382017-04-27 15:03:42766 // Register the CNG payload type if it's been added, don't do anything if CNG
767 // is removed. Payload types must not be redefined.
768 if (new_config.send_codec_spec->cng_payload_type) {
Sebastian Jansson35cf9e72019-10-04 07:30:32769 RegisterCngPayloadType(*new_config.send_codec_spec->cng_payload_type,
770 new_config.send_codec_spec->format.clockrate_hz);
ossu3b9ff382017-04-27 15:03:42771 }
772
ossu20a4b3f2017-04-27 09:08:52773 // Wrap or unwrap the encoder in an AudioEncoderCNG.
Sebastian Jansson35cf9e72019-10-04 07:30:32774 channel_send_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder_ptr) {
775 std::unique_ptr<AudioEncoder> old_encoder(std::move(*encoder_ptr));
776 auto sub_encoders = old_encoder->ReclaimContainedEncoders();
777 if (!sub_encoders.empty()) {
778 // Replace enc with its sub encoder. We need to put the sub
779 // encoder in a temporary first, since otherwise the old value
780 // of enc would be destroyed before the new value got assigned,
781 // which would be bad since the new value is a part of the old
782 // value.
783 auto tmp = std::move(sub_encoders[0]);
784 old_encoder = std::move(tmp);
785 }
786 if (new_config.send_codec_spec->cng_payload_type) {
787 AudioEncoderCngConfig config;
788 config.speech_encoder = std::move(old_encoder);
789 config.num_channels = config.speech_encoder->NumChannels();
790 config.payload_type = *new_config.send_codec_spec->cng_payload_type;
791 config.vad_mode = Vad::kVadNormal;
792 *encoder_ptr = CreateComfortNoiseEncoder(std::move(config));
793 } else {
794 *encoder_ptr = std::move(old_encoder);
795 }
796 });
ossu20a4b3f2017-04-27 09:08:52797}
798
799void AudioSendStream::ReconfigureBitrateObserver(
ossu20a4b3f2017-04-27 09:08:52800 const webrtc::AudioSendStream::Config& new_config) {
801 // Since the Config's default is for both of these to be -1, this test will
802 // allow us to configure the bitrate observer if the new config has bitrate
803 // limits set, but would only have us call RemoveBitrateObserver if we were
804 // previously configured with bitrate limits.
Sebastian Jansson35cf9e72019-10-04 07:30:32805 if (config_.min_bitrate_bps == new_config.min_bitrate_bps &&
806 config_.max_bitrate_bps == new_config.max_bitrate_bps &&
807 config_.bitrate_priority == new_config.bitrate_priority &&
Jakob Ivarsson47a03e82020-11-23 14:05:44808 TransportSeqNumId(config_) == TransportSeqNumId(new_config) &&
Jakob Ivarssond14525e2020-03-06 08:49:29809 config_.audio_network_adaptor_config ==
810 new_config.audio_network_adaptor_config) {
ossu20a4b3f2017-04-27 09:08:52811 return;
812 }
813
Sebastian Janssonf23131f2019-10-03 08:03:55814 if (!new_config.has_dscp && new_config.min_bitrate_bps != -1 &&
Sebastian Janssoncd0eedb2019-10-10 11:52:26815 new_config.max_bitrate_bps != -1 && TransportSeqNumId(new_config) != 0) {
Sebastian Jansson35cf9e72019-10-04 07:30:32816 rtp_transport_->AccountForAudioPacketsInPacedSender(true);
Sebastian Janssonc3eb9fd2020-01-29 16:42:52817 if (send_side_bwe_with_overhead_)
818 rtp_transport_->IncludeOverheadInPacedSender();
Artem Titova2088612021-02-03 12:33:28819 // We may get a callback immediately as the observer is registered, so
820 // make sure the bitrate limits in config_ are up-to-date.
821 config_.min_bitrate_bps = new_config.min_bitrate_bps;
822 config_.max_bitrate_bps = new_config.max_bitrate_bps;
Sebastian Jansson35cf9e72019-10-04 07:30:32823
Artem Titova2088612021-02-03 12:33:28824 config_.bitrate_priority = new_config.bitrate_priority;
825 ConfigureBitrateObserver();
Sebastian Jansson35cf9e72019-10-04 07:30:32826 rtp_rtcp_module_->SetAsPartOfAllocation(true);
ossu20a4b3f2017-04-27 09:08:52827 } else {
Sebastian Jansson35cf9e72019-10-04 07:30:32828 rtp_transport_->AccountForAudioPacketsInPacedSender(false);
829 RemoveBitrateObserver();
830 rtp_rtcp_module_->SetAsPartOfAllocation(false);
ossu20a4b3f2017-04-27 09:08:52831 }
832}
833
Sebastian Jansson8672cac2019-03-01 14:57:55834void AudioSendStream::ConfigureBitrateObserver() {
835 // This either updates the current observer or adds a new observer.
836 // TODO(srte): Add overhead compensation here.
Daniel Lee93562522019-05-03 12:40:13837 auto constraints = GetMinMaxBitrateConstraints();
Artem Titova2088612021-02-03 12:33:28838 RTC_DCHECK(constraints.has_value());
Daniel Lee93562522019-05-03 12:40:13839
Sebastian Jansson0429f782019-10-03 16:32:45840 DataRate priority_bitrate = allocation_settings_.priority_bitrate;
Sebastian Janssonf23131f2019-10-03 08:03:55841 if (send_side_bwe_with_overhead_) {
Sebastian Jansson0429f782019-10-03 16:32:45842 if (use_legacy_overhead_calculation_) {
843 // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12)
844 constexpr int kOverheadPerPacket = 20 + 8 + 10 + 12;
Danil Chapovalov0c626af2020-02-10 10:16:00845 const TimeDelta kMinPacketDuration = TimeDelta::Millis(20);
Sebastian Jansson0429f782019-10-03 16:32:45846 DataRate max_overhead =
Danil Chapovalovcad3e0e2020-02-17 17:46:07847 DataSize::Bytes(kOverheadPerPacket) / kMinPacketDuration;
Sebastian Jansson0429f782019-10-03 16:32:45848 priority_bitrate += max_overhead;
849 } else {
850 RTC_DCHECK(frame_length_range_);
Erik Språng04e1bab2020-05-07 16:18:32851 const DataSize overhead_per_packet =
Danil Chapovalovcad3e0e2020-02-17 17:46:07852 DataSize::Bytes(total_packet_overhead_bytes_);
Erik Språng04e1bab2020-05-07 16:18:32853 DataRate min_overhead = overhead_per_packet / frame_length_range_->second;
Jakob Ivarsson01ab0842020-03-06 08:59:56854 priority_bitrate += min_overhead;
Sebastian Jansson0429f782019-10-03 16:32:45855 }
Sebastian Janssonf23131f2019-10-03 08:03:55856 }
Sebastian Janssonf23131f2019-10-03 08:03:55857 if (allocation_settings_.priority_bitrate_raw)
858 priority_bitrate = *allocation_settings_.priority_bitrate_raw;
859
Markus Handell3907e7b2021-06-01 07:07:20860 rtp_transport_queue_->PostTask([this, constraints, priority_bitrate,
861 config_bitrate_priority =
862 config_.bitrate_priority] {
863 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Artem Titova2088612021-02-03 12:33:28864 bitrate_allocator_->AddObserver(
865 this,
866 MediaStreamAllocationConfig{
867 constraints->min.bps<uint32_t>(), constraints->max.bps<uint32_t>(),
868 0, priority_bitrate.bps(), true,
869 allocation_settings_.bitrate_priority.value_or(
870 config_bitrate_priority)});
871 });
Jakob Ivarssond14525e2020-03-06 08:49:29872 registered_with_allocator_ = true;
ossu20a4b3f2017-04-27 09:08:52873}
874
875void AudioSendStream::RemoveBitrateObserver() {
Artem Titova2088612021-02-03 12:33:28876 registered_with_allocator_ = false;
Niels Möllerc572ff32018-11-07 07:43:50877 rtc::Event thread_sync_event;
Markus Handell3907e7b2021-06-01 07:07:20878 rtp_transport_queue_->PostTask([this, &thread_sync_event] {
879 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
ossu20a4b3f2017-04-27 09:08:52880 bitrate_allocator_->RemoveObserver(this);
881 thread_sync_event.Set();
882 });
883 thread_sync_event.Wait(rtc::Event::kForever);
884}
885
Artem Titova2088612021-02-03 12:33:28886absl::optional<AudioSendStream::TargetAudioBitrateConstraints>
Daniel Lee93562522019-05-03 12:40:13887AudioSendStream::GetMinMaxBitrateConstraints() const {
Artem Titova2088612021-02-03 12:33:28888 if (config_.min_bitrate_bps < 0 || config_.max_bitrate_bps < 0) {
889 RTC_LOG(LS_WARNING) << "Config is invalid: min_bitrate_bps="
890 << config_.min_bitrate_bps
891 << "; max_bitrate_bps=" << config_.max_bitrate_bps
892 << "; both expected greater or equal to 0";
893 return absl::nullopt;
894 }
Daniel Lee93562522019-05-03 12:40:13895 TargetAudioBitrateConstraints constraints{
Danil Chapovalovcad3e0e2020-02-17 17:46:07896 DataRate::BitsPerSec(config_.min_bitrate_bps),
897 DataRate::BitsPerSec(config_.max_bitrate_bps)};
Daniel Lee93562522019-05-03 12:40:13898
899 // If bitrates were explicitly overriden via field trial, use those values.
Sebastian Janssonf23131f2019-10-03 08:03:55900 if (allocation_settings_.min_bitrate)
901 constraints.min = *allocation_settings_.min_bitrate;
902 if (allocation_settings_.max_bitrate)
903 constraints.max = *allocation_settings_.max_bitrate;
Daniel Lee93562522019-05-03 12:40:13904
Sebastian Jansson62aee932019-10-02 10:27:06905 RTC_DCHECK_GE(constraints.min, DataRate::Zero());
906 RTC_DCHECK_GE(constraints.max, DataRate::Zero());
Artem Titova2088612021-02-03 12:33:28907 if (constraints.max < constraints.min) {
908 RTC_LOG(LS_WARNING) << "TargetAudioBitrateConstraints::max is less than "
909 << "TargetAudioBitrateConstraints::min";
910 return absl::nullopt;
911 }
Sebastian Janssonf23131f2019-10-03 08:03:55912 if (send_side_bwe_with_overhead_) {
Sebastian Jansson62aee932019-10-02 10:27:06913 if (use_legacy_overhead_calculation_) {
914 // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12)
Danil Chapovalovcad3e0e2020-02-17 17:46:07915 const DataSize kOverheadPerPacket = DataSize::Bytes(20 + 8 + 10 + 12);
Sebastian Jansson62aee932019-10-02 10:27:06916 const TimeDelta kMaxFrameLength =
Danil Chapovalov0c626af2020-02-10 10:16:00917 TimeDelta::Millis(60); // Based on Opus spec
Sebastian Jansson62aee932019-10-02 10:27:06918 const DataRate kMinOverhead = kOverheadPerPacket / kMaxFrameLength;
919 constraints.min += kMinOverhead;
920 constraints.max += kMinOverhead;
921 } else {
Artem Titova2088612021-02-03 12:33:28922 if (!frame_length_range_.has_value()) {
923 RTC_LOG(LS_WARNING) << "frame_length_range_ is not set";
924 return absl::nullopt;
925 }
Sebastian Jansson62aee932019-10-02 10:27:06926 const DataSize kOverheadPerPacket =
Danil Chapovalovcad3e0e2020-02-17 17:46:07927 DataSize::Bytes(total_packet_overhead_bytes_);
Sebastian Jansson62aee932019-10-02 10:27:06928 constraints.min += kOverheadPerPacket / frame_length_range_->second;
929 constraints.max += kOverheadPerPacket / frame_length_range_->first;
930 }
Daniel Lee93562522019-05-03 12:40:13931 }
932 return constraints;
933}
934
ossu3b9ff382017-04-27 15:03:42935void AudioSendStream::RegisterCngPayloadType(int payload_type,
936 int clockrate_hz) {
Niels Mölleree5ccbc2019-03-06 15:47:29937 channel_send_->RegisterCngPayloadType(payload_type, clockrate_hz);
ossu3b9ff382017-04-27 15:03:42938}
Artem Titova2088612021-02-03 12:33:28939
940void AudioSendStream::UpdateCachedTargetAudioBitrateConstraints() {
941 absl::optional<AudioSendStream::TargetAudioBitrateConstraints>
942 new_constraints = GetMinMaxBitrateConstraints();
943 if (!new_constraints.has_value()) {
944 return;
945 }
Markus Handell3907e7b2021-06-01 07:07:20946 rtp_transport_queue_->PostTask([this, new_constraints]() {
947 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Artem Titova2088612021-02-03 12:33:28948 cached_constraints_ = new_constraints;
949 });
950}
951
solenbergc7a8b082015-10-16 21:35:07952} // namespace internal
953} // namespace webrtc