blob: 6aa0ec940f7ce013440e3628711f2ab8ecdb8004 [file] [log] [blame]
solenbergc7a8b082015-10-16 21:35:071/*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 04:47:3111#include "audio/audio_send_stream.h"
solenbergc7a8b082015-10-16 21:35:0712
Mirko Bonadei317a1f02019-09-17 15:06:1813#include <memory>
solenbergc7a8b082015-10-16 21:35:0714#include <string>
ossu20a4b3f2017-04-27 09:08:5215#include <utility>
16#include <vector>
solenbergc7a8b082015-10-16 21:35:0717
Yves Gerey988cc082018-10-23 10:03:0118#include "api/audio_codecs/audio_encoder.h"
19#include "api/audio_codecs/audio_encoder_factory.h"
20#include "api/audio_codecs/audio_format.h"
21#include "api/call/transport.h"
Steve Anton10542f22019-01-11 17:11:0022#include "api/crypto/frame_encryptor_interface.h"
Artem Titov741daaf2019-03-21 13:37:3623#include "api/function_view.h"
Danil Chapovalov83bbe912019-08-07 10:24:5324#include "api/rtc_event_log/rtc_event_log.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3125#include "audio/audio_state.h"
Yves Gerey988cc082018-10-23 10:03:0126#include "audio/channel_send.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3127#include "audio/conversion.h"
Yves Gerey988cc082018-10-23 10:03:0128#include "call/rtp_config.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3129#include "call/rtp_transport_controller_send_interface.h"
Yves Gerey988cc082018-10-23 10:03:0130#include "common_audio/vad/include/vad.h"
Oskar Sundbom56ef3052018-10-30 15:11:0231#include "logging/rtc_event_log/events/rtc_event_audio_send_stream_config.h"
Oskar Sundbom56ef3052018-10-30 15:11:0232#include "logging/rtc_event_log/rtc_stream_config.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3133#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h"
Philipp Hanckeedcd9662020-06-24 10:52:4234#include "modules/audio_coding/codecs/red/audio_encoder_copy_red.h"
Yves Gerey988cc082018-10-23 10:03:0135#include "modules/audio_processing/include/audio_processing.h"
Sebastian Jansson6298b562020-01-14 16:55:1936#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3137#include "rtc_base/checks.h"
38#include "rtc_base/event.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3139#include "rtc_base/logging.h"
Jonas Olssonabbe8412018-04-03 11:40:0540#include "rtc_base/strings/audio_format_to_string.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3141#include "rtc_base/task_queue.h"
solenbergc7a8b082015-10-16 21:35:0742
43namespace webrtc {
Fredrik Solenberg8f5787a2018-01-11 12:52:3044namespace {
elad.alond12a8e12017-03-23 18:04:4845
Oskar Sundbom56ef3052018-10-30 15:11:0246void UpdateEventLogStreamConfig(RtcEventLog* event_log,
47 const AudioSendStream::Config& config,
48 const AudioSendStream::Config* old_config) {
49 using SendCodecSpec = AudioSendStream::Config::SendCodecSpec;
50 // Only update if any of the things we log have changed.
51 auto payload_types_equal = [](const absl::optional<SendCodecSpec>& a,
52 const absl::optional<SendCodecSpec>& b) {
53 if (a.has_value() && b.has_value()) {
54 return a->format.name == b->format.name &&
55 a->payload_type == b->payload_type;
56 }
57 return !a.has_value() && !b.has_value();
58 };
59
60 if (old_config && config.rtp.ssrc == old_config->rtp.ssrc &&
61 config.rtp.extensions == old_config->rtp.extensions &&
62 payload_types_equal(config.send_codec_spec,
63 old_config->send_codec_spec)) {
64 return;
65 }
66
Mirko Bonadei317a1f02019-09-17 15:06:1867 auto rtclog_config = std::make_unique<rtclog::StreamConfig>();
Oskar Sundbom56ef3052018-10-30 15:11:0268 rtclog_config->local_ssrc = config.rtp.ssrc;
69 rtclog_config->rtp_extensions = config.rtp.extensions;
70 if (config.send_codec_spec) {
71 rtclog_config->codecs.emplace_back(config.send_codec_spec->format.name,
72 config.send_codec_spec->payload_type, 0);
73 }
Mirko Bonadei317a1f02019-09-17 15:06:1874 event_log->Log(std::make_unique<RtcEventAudioSendStreamConfig>(
Oskar Sundbom56ef3052018-10-30 15:11:0275 std::move(rtclog_config)));
76}
ossu20a4b3f2017-04-27 09:08:5277} // namespace
78
Sebastian Janssonf23131f2019-10-03 08:03:5579constexpr char AudioAllocationConfig::kKey[];
80
81std::unique_ptr<StructParametersParser> AudioAllocationConfig::Parser() {
82 return StructParametersParser::Create( //
83 "min", &min_bitrate, //
84 "max", &max_bitrate, //
85 "prio_rate", &priority_bitrate, //
86 "prio_rate_raw", &priority_bitrate_raw, //
87 "rate_prio", &bitrate_priority);
88}
89
Jonas Orelanda943e732022-03-16 12:50:5890AudioAllocationConfig::AudioAllocationConfig(
91 const WebRtcKeyValueConfig& field_trials) {
92 Parser()->Parse(field_trials.Lookup(kKey));
Sebastian Janssonf23131f2019-10-03 08:03:5593 if (priority_bitrate_raw && !priority_bitrate.IsZero()) {
94 RTC_LOG(LS_WARNING) << "'priority_bitrate' and '_raw' are mutually "
95 "exclusive but both were configured.";
96 }
97}
98
99namespace internal {
solenberg566ef242015-11-06 23:34:49100AudioSendStream::AudioSendStream(
Sebastian Jansson977b3352019-03-04 16:43:34101 Clock* clock,
solenberg566ef242015-11-06 23:34:49102 const webrtc::AudioSendStream::Config& config,
Stefan Holmerb86d4e42015-12-07 09:26:18103 const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
Sebastian Jansson44dd9f22019-03-08 13:50:30104 TaskQueueFactory* task_queue_factory,
Niels Möller7d76a312018-10-26 10:57:07105 RtpTransportControllerSendInterface* rtp_transport,
Niels Möller67b011d2018-10-22 11:00:40106 BitrateAllocatorInterface* bitrate_allocator,
michaelt9332b7d2016-11-30 15:51:13107 RtcEventLog* event_log,
ossuc3d4b482017-05-23 13:07:11108 RtcpRttStats* rtcp_rtt_stats,
Jonas Orelanda943e732022-03-16 12:50:58109 const absl::optional<RtpState>& suspended_rtp_state,
110 const WebRtcKeyValueConfig& field_trials)
111 : AudioSendStream(
112 clock,
113 config,
114 audio_state,
115 task_queue_factory,
116 rtp_transport,
117 bitrate_allocator,
118 event_log,
119 suspended_rtp_state,
120 voe::CreateChannelSend(clock,
121 task_queue_factory,
122 config.send_transport,
123 rtcp_rtt_stats,
124 event_log,
125 config.frame_encryptor,
126 config.crypto_options,
127 config.rtp.extmap_allow_mixed,
128 config.rtcp_report_interval_ms,
129 config.rtp.ssrc,
130 config.frame_transformer,
131 rtp_transport->transport_feedback_observer(),
132 field_trials),
133 field_trials) {}
Fredrik Solenberg8f5787a2018-01-11 12:52:30134
135AudioSendStream::AudioSendStream(
Sebastian Jansson977b3352019-03-04 16:43:34136 Clock* clock,
Fredrik Solenberg8f5787a2018-01-11 12:52:30137 const webrtc::AudioSendStream::Config& config,
138 const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
Sebastian Jansson44dd9f22019-03-08 13:50:30139 TaskQueueFactory* task_queue_factory,
Niels Möller7d76a312018-10-26 10:57:07140 RtpTransportControllerSendInterface* rtp_transport,
Niels Möller67b011d2018-10-22 11:00:40141 BitrateAllocatorInterface* bitrate_allocator,
Fredrik Solenberg8f5787a2018-01-11 12:52:30142 RtcEventLog* event_log,
Danil Chapovalovb9b146c2018-06-15 10:28:07143 const absl::optional<RtpState>& suspended_rtp_state,
Jonas Orelanda943e732022-03-16 12:50:58144 std::unique_ptr<voe::ChannelSendInterface> channel_send,
145 const WebRtcKeyValueConfig& field_trials)
Sebastian Jansson977b3352019-03-04 16:43:34146 : clock_(clock),
Jonas Orelanda943e732022-03-16 12:50:58147 field_trials_(field_trials),
Markus Handell3907e7b2021-06-01 07:07:20148 rtp_transport_queue_(rtp_transport->GetWorkerQueue()),
Sebastian Janssonf23131f2019-10-03 08:03:55149 allocate_audio_without_feedback_(
Jonas Orelanda943e732022-03-16 12:50:58150 field_trials_.IsEnabled("WebRTC-Audio-ABWENoTWCC")),
Sebastian Janssonf23131f2019-10-03 08:03:55151 enable_audio_alr_probing_(
Jonas Orelanda943e732022-03-16 12:50:58152 !field_trials_.IsDisabled("WebRTC-Audio-AlrProbing")),
Sebastian Janssonf23131f2019-10-03 08:03:55153 send_side_bwe_with_overhead_(
Jonas Orelanda943e732022-03-16 12:50:58154 !field_trials_.IsDisabled("WebRTC-SendSideBwe-WithOverhead")),
155 allocation_settings_(field_trials_),
Bjorn A Mellem7a9a0922019-11-26 17:19:40156 config_(Config(/*send_transport=*/nullptr)),
mflodman86cc6ff2016-07-26 11:44:06157 audio_state_(audio_state),
Niels Möllerdced9f62018-11-19 09:27:07158 channel_send_(std::move(channel_send)),
ossu20a4b3f2017-04-27 09:08:52159 event_log_(event_log),
Sebastian Jansson62aee932019-10-02 10:27:06160 use_legacy_overhead_calculation_(
Jonas Orelanda943e732022-03-16 12:50:58161 field_trials_.IsEnabled("WebRTC-Audio-LegacyOverhead")),
michaeltf4caaab2017-01-17 07:55:07162 bitrate_allocator_(bitrate_allocator),
Niels Möller7d76a312018-10-26 10:57:07163 rtp_transport_(rtp_transport),
Sebastian Jansson6298b562020-01-14 16:55:19164 rtp_rtcp_module_(channel_send_->GetRtpRtcp()),
Sam Zackrissonff058162018-11-20 16:15:13165 suspended_rtp_state_(suspended_rtp_state) {
Jonas Olsson24ea8222018-01-25 09:14:29166 RTC_LOG(LS_INFO) << "AudioSendStream: " << config.rtp.ssrc;
Markus Handell3907e7b2021-06-01 07:07:20167 RTC_DCHECK(rtp_transport_queue_);
Fredrik Solenberg8f5787a2018-01-11 12:52:30168 RTC_DCHECK(audio_state_);
Niels Möllerdced9f62018-11-19 09:27:07169 RTC_DCHECK(channel_send_);
Fredrik Solenberg8f5787a2018-01-11 12:52:30170 RTC_DCHECK(bitrate_allocator_);
Sebastian Jansson0b698262019-03-07 08:17:19171 RTC_DCHECK(rtp_transport);
172
ossuc3d4b482017-05-23 13:07:11173 RTC_DCHECK(rtp_rtcp_module_);
mflodman3d7db262016-04-29 07:57:13174
Artem Titova2088612021-02-03 12:33:28175 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Sebastian Jansson35cf9e72019-10-04 07:30:32176 ConfigureStream(config, true);
Artem Titova2088612021-02-03 12:33:28177 UpdateCachedTargetAudioBitrateConstraints();
Sebastian Janssonc01367d2019-04-08 13:20:44178 pacer_thread_checker_.Detach();
solenbergc7a8b082015-10-16 21:35:07179}
180
181AudioSendStream::~AudioSendStream() {
Artem Titova2088612021-02-03 12:33:28182 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Jonas Olsson24ea8222018-01-25 09:14:29183 RTC_LOG(LS_INFO) << "~AudioSendStream: " << config_.rtp.ssrc;
Fredrik Solenberg2a877972017-12-15 15:42:15184 RTC_DCHECK(!sending_);
Sebastian Jansson0a6510d2019-10-04 07:31:08185 channel_send_->ResetSenderCongestionControlObjects();
Sebastian Jansson8672cac2019-03-01 14:57:55186 // Blocking call to synchronize state with worker queue to ensure that there
187 // are no pending tasks left that keeps references to audio.
188 rtc::Event thread_sync_event;
Markus Handell3907e7b2021-06-01 07:07:20189 rtp_transport_queue_->PostTask([&] { thread_sync_event.Set(); });
Sebastian Jansson8672cac2019-03-01 14:57:55190 thread_sync_event.Wait(rtc::Event::kForever);
solenbergc7a8b082015-10-16 21:35:07191}
192
eladalonabbc4302017-07-26 09:09:44193const webrtc::AudioSendStream::Config& AudioSendStream::GetConfig() const {
Artem Titova2088612021-02-03 12:33:28194 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
eladalonabbc4302017-07-26 09:09:44195 return config_;
196}
197
ossu20a4b3f2017-04-27 09:08:52198void AudioSendStream::Reconfigure(
199 const webrtc::AudioSendStream::Config& new_config) {
Artem Titova2088612021-02-03 12:33:28200 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Sebastian Jansson35cf9e72019-10-04 07:30:32201 ConfigureStream(new_config, false);
ossu20a4b3f2017-04-27 09:08:52202}
203
Alex Narestcedd3512017-12-07 19:54:55204AudioSendStream::ExtensionIds AudioSendStream::FindExtensionIds(
205 const std::vector<RtpExtension>& extensions) {
206 ExtensionIds ids;
207 for (const auto& extension : extensions) {
208 if (extension.uri == RtpExtension::kAudioLevelUri) {
209 ids.audio_level = extension.id;
Sebastian Jansson71c6b562019-08-14 09:31:02210 } else if (extension.uri == RtpExtension::kAbsSendTimeUri) {
211 ids.abs_send_time = extension.id;
Alex Narestcedd3512017-12-07 19:54:55212 } else if (extension.uri == RtpExtension::kTransportSequenceNumberUri) {
213 ids.transport_sequence_number = extension.id;
Steve Antonbb50ce52018-03-26 17:24:32214 } else if (extension.uri == RtpExtension::kMidUri) {
215 ids.mid = extension.id;
Amit Hilbuch77938e62018-12-21 17:23:38216 } else if (extension.uri == RtpExtension::kRidUri) {
217 ids.rid = extension.id;
218 } else if (extension.uri == RtpExtension::kRepairedRidUri) {
219 ids.repaired_rid = extension.id;
Minyue Li74dadc12020-03-05 10:33:13220 } else if (extension.uri == RtpExtension::kAbsoluteCaptureTimeUri) {
221 ids.abs_capture_time = extension.id;
Alex Narestcedd3512017-12-07 19:54:55222 }
223 }
224 return ids;
225}
226
Sebastian Jansson470a5ea2019-01-23 11:37:49227int AudioSendStream::TransportSeqNumId(const AudioSendStream::Config& config) {
228 return FindExtensionIds(config.rtp.extensions).transport_sequence_number;
229}
230
ossu20a4b3f2017-04-27 09:08:52231void AudioSendStream::ConfigureStream(
ossu20a4b3f2017-04-27 09:08:52232 const webrtc::AudioSendStream::Config& new_config,
233 bool first_time) {
Jonas Olsson24ea8222018-01-25 09:14:29234 RTC_LOG(LS_INFO) << "AudioSendStream::ConfigureStream: "
235 << new_config.ToString();
Sebastian Jansson35cf9e72019-10-04 07:30:32236 UpdateEventLogStreamConfig(event_log_, new_config,
237 first_time ? nullptr : &config_);
Oskar Sundbom56ef3052018-10-30 15:11:02238
Sebastian Jansson35cf9e72019-10-04 07:30:32239 const auto& old_config = config_;
ossu20a4b3f2017-04-27 09:08:52240
Niels Möllere9771992018-11-26 09:55:07241 // Configuration parameters which cannot be changed.
242 RTC_DCHECK(first_time ||
243 old_config.send_transport == new_config.send_transport);
Erik Språng70efdde2019-08-21 11:36:20244 RTC_DCHECK(first_time || old_config.rtp.ssrc == new_config.rtp.ssrc);
Sebastian Jansson35cf9e72019-10-04 07:30:32245 if (suspended_rtp_state_ && first_time) {
246 rtp_rtcp_module_->SetRtpState(*suspended_rtp_state_);
ossu20a4b3f2017-04-27 09:08:52247 }
248 if (first_time || old_config.rtp.c_name != new_config.rtp.c_name) {
Sebastian Jansson35cf9e72019-10-04 07:30:32249 channel_send_->SetRTCP_CNAME(new_config.rtp.c_name);
ossu20a4b3f2017-04-27 09:08:52250 }
ossu20a4b3f2017-04-27 09:08:52251
Benjamin Wright84583f62018-10-04 21:22:34252 // Enable the frame encryptor if a new frame encryptor has been provided.
253 if (first_time || new_config.frame_encryptor != old_config.frame_encryptor) {
Sebastian Jansson35cf9e72019-10-04 07:30:32254 channel_send_->SetFrameEncryptor(new_config.frame_encryptor);
Benjamin Wright84583f62018-10-04 21:22:34255 }
256
Johannes Kron9190b822018-10-29 10:22:05257 if (first_time ||
Marina Ciocead2aa8f92020-03-31 09:29:56258 new_config.frame_transformer != old_config.frame_transformer) {
259 channel_send_->SetEncoderToPacketizerFrameTransformer(
260 new_config.frame_transformer);
261 }
262
263 if (first_time ||
Johannes Kron9190b822018-10-29 10:22:05264 new_config.rtp.extmap_allow_mixed != old_config.rtp.extmap_allow_mixed) {
Sebastian Jansson6298b562020-01-14 16:55:19265 rtp_rtcp_module_->SetExtmapAllowMixed(new_config.rtp.extmap_allow_mixed);
Johannes Kron9190b822018-10-29 10:22:05266 }
267
Alex Narestcedd3512017-12-07 19:54:55268 const ExtensionIds old_ids = FindExtensionIds(old_config.rtp.extensions);
269 const ExtensionIds new_ids = FindExtensionIds(new_config.rtp.extensions);
Yves Gerey17048012019-07-26 15:49:52270
ossu20a4b3f2017-04-27 09:08:52271 // Audio level indication
272 if (first_time || new_ids.audio_level != old_ids.audio_level) {
Sebastian Jansson35cf9e72019-10-04 07:30:32273 channel_send_->SetSendAudioLevelIndicationStatus(new_ids.audio_level != 0,
274 new_ids.audio_level);
ossu20a4b3f2017-04-27 09:08:52275 }
Sebastian Jansson71c6b562019-08-14 09:31:02276
277 if (first_time || new_ids.abs_send_time != old_ids.abs_send_time) {
Danil Chapovalov723b35f2021-10-07 17:12:32278 absl::string_view uri = AbsoluteSendTime::Uri();
279 rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(uri);
Sebastian Jansson71c6b562019-08-14 09:31:02280 if (new_ids.abs_send_time) {
Danil Chapovalov723b35f2021-10-07 17:12:32281 rtp_rtcp_module_->RegisterRtpHeaderExtension(uri, new_ids.abs_send_time);
Sebastian Jansson71c6b562019-08-14 09:31:02282 }
283 }
284
Sebastian Jansson8d9c5402017-11-15 16:22:16285 bool transport_seq_num_id_changed =
286 new_ids.transport_sequence_number != old_ids.transport_sequence_number;
Sebastian Jansson35cf9e72019-10-04 07:30:32287 if (first_time ||
288 (transport_seq_num_id_changed && !allocate_audio_without_feedback_)) {
ossu1129df22017-06-30 08:38:56289 if (!first_time) {
Sebastian Jansson35cf9e72019-10-04 07:30:32290 channel_send_->ResetSenderCongestionControlObjects();
ossu20a4b3f2017-04-27 09:08:52291 }
292
Sebastian Jansson8d9c5402017-11-15 16:22:16293 RtcpBandwidthObserver* bandwidth_observer = nullptr;
Sebastian Jansson470a5ea2019-01-23 11:37:49294
Jakob Ivarsson47a03e82020-11-23 14:05:44295 if (!allocate_audio_without_feedback_ &&
Sebastian Janssonf23131f2019-10-03 08:03:55296 new_ids.transport_sequence_number != 0) {
Sebastian Jansson6298b562020-01-14 16:55:19297 rtp_rtcp_module_->RegisterRtpHeaderExtension(
Danil Chapovalovd0321c52021-09-14 10:58:51298 TransportSequenceNumber::Uri(), new_ids.transport_sequence_number);
Sebastian Jansson8d9c5402017-11-15 16:22:16299 // Probing in application limited region is only used in combination with
300 // send side congestion control, wich depends on feedback packets which
301 // requires transport sequence numbers to be enabled.
Sebastian Jansson0a6510d2019-10-04 07:31:08302 // Optionally request ALR probing but do not override any existing
303 // request from other streams.
304 if (enable_audio_alr_probing_) {
305 rtp_transport_->EnablePeriodicAlrProbing(true);
Niels Möller7d76a312018-10-26 10:57:07306 }
Sebastian Jansson0a6510d2019-10-04 07:31:08307 bandwidth_observer = rtp_transport_->GetBandwidthObserver();
ossu20a4b3f2017-04-27 09:08:52308 }
Sebastian Jansson0a6510d2019-10-04 07:31:08309 channel_send_->RegisterSenderCongestionControlObjects(rtp_transport_,
310 bandwidth_observer);
ossu20a4b3f2017-04-27 09:08:52311 }
Steve Antonbb50ce52018-03-26 17:24:32312 // MID RTP header extension.
Steve Anton003930a2018-03-29 19:37:21313 if ((first_time || new_ids.mid != old_ids.mid ||
314 new_config.rtp.mid != old_config.rtp.mid) &&
315 new_ids.mid != 0 && !new_config.rtp.mid.empty()) {
Danil Chapovalovd0321c52021-09-14 10:58:51316 rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpMid::Uri(), new_ids.mid);
Sebastian Jansson6298b562020-01-14 16:55:19317 rtp_rtcp_module_->SetMid(new_config.rtp.mid);
Steve Antonbb50ce52018-03-26 17:24:32318 }
319
Minyue Li74dadc12020-03-05 10:33:13320 if (first_time || new_ids.abs_capture_time != old_ids.abs_capture_time) {
Danil Chapovalov723b35f2021-10-07 17:12:32321 absl::string_view uri = AbsoluteCaptureTimeExtension::Uri();
322 rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(uri);
Minyue Li74dadc12020-03-05 10:33:13323 if (new_ids.abs_capture_time) {
Danil Chapovalov723b35f2021-10-07 17:12:32324 rtp_rtcp_module_->RegisterRtpHeaderExtension(uri,
325 new_ids.abs_capture_time);
Minyue Li74dadc12020-03-05 10:33:13326 }
327 }
328
Sebastian Jansson35cf9e72019-10-04 07:30:32329 if (!ReconfigureSendCodec(new_config)) {
Mirko Bonadei675513b2017-11-09 10:09:25330 RTC_LOG(LS_ERROR) << "Failed to set up send codec state.";
ossu20a4b3f2017-04-27 09:08:52331 }
332
Erik Språng04e1bab2020-05-07 16:18:32333 // Set currently known overhead (used in ANA, opus only).
334 {
Markus Handell62872802020-07-06 13:15:07335 MutexLock lock(&overhead_per_packet_lock_);
Erik Språng04e1bab2020-05-07 16:18:32336 UpdateOverheadForEncoder();
337 }
338
Jakob Ivarssond14525e2020-03-06 08:49:29339 channel_send_->CallEncoder([this](AudioEncoder* encoder) {
Artem Titova2088612021-02-03 12:33:28340 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Jakob Ivarssond14525e2020-03-06 08:49:29341 if (!encoder) {
342 return;
343 }
Artem Titova2088612021-02-03 12:33:28344 frame_length_range_ = encoder->GetFrameLengthRange();
345 UpdateCachedTargetAudioBitrateConstraints();
Jakob Ivarssond14525e2020-03-06 08:49:29346 });
347
Sebastian Jansson35cf9e72019-10-04 07:30:32348 if (sending_) {
349 ReconfigureBitrateObserver(new_config);
Oskar Sundbomf85e31b2017-12-20 15:38:09350 }
Artem Titova2088612021-02-03 12:33:28351
Sebastian Jansson35cf9e72019-10-04 07:30:32352 config_ = new_config;
Artem Titova2088612021-02-03 12:33:28353 if (!first_time) {
354 UpdateCachedTargetAudioBitrateConstraints();
355 }
ossu20a4b3f2017-04-27 09:08:52356}
357
solenberg3a941542015-11-16 15:34:50358void AudioSendStream::Start() {
Sebastian Jansson8672cac2019-03-01 14:57:55359 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Fredrik Solenberg2a877972017-12-15 15:42:15360 if (sending_) {
361 return;
362 }
Sebastian Janssonf23131f2019-10-03 08:03:55363 if (!config_.has_dscp && config_.min_bitrate_bps != -1 &&
364 config_.max_bitrate_bps != -1 &&
Sebastian Janssoncd0eedb2019-10-10 11:52:26365 (allocate_audio_without_feedback_ || TransportSeqNumId(config_) != 0)) {
Erik Språngaa59eca2019-07-24 12:52:55366 rtp_transport_->AccountForAudioPacketsInPacedSender(true);
Sebastian Janssonc3eb9fd2020-01-29 16:42:52367 if (send_side_bwe_with_overhead_)
368 rtp_transport_->IncludeOverheadInPacedSender();
Sebastian Janssonb6863962018-10-10 08:23:13369 rtp_rtcp_module_->SetAsPartOfAllocation(true);
Artem Titova2088612021-02-03 12:33:28370 ConfigureBitrateObserver();
Sebastian Janssonb6863962018-10-10 08:23:13371 } else {
372 rtp_rtcp_module_->SetAsPartOfAllocation(false);
mflodman86cc6ff2016-07-26 11:44:06373 }
Niels Möllerdced9f62018-11-19 09:27:07374 channel_send_->StartSend();
Fredrik Solenberg2a877972017-12-15 15:42:15375 sending_ = true;
376 audio_state()->AddSendingStream(this, encoder_sample_rate_hz_,
377 encoder_num_channels_);
solenberg3a941542015-11-16 15:34:50378}
379
380void AudioSendStream::Stop() {
Artem Titova2088612021-02-03 12:33:28381 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Fredrik Solenberg2a877972017-12-15 15:42:15382 if (!sending_) {
383 return;
384 }
385
ossu20a4b3f2017-04-27 09:08:52386 RemoveBitrateObserver();
Niels Möllerdced9f62018-11-19 09:27:07387 channel_send_->StopSend();
Fredrik Solenberg2a877972017-12-15 15:42:15388 sending_ = false;
389 audio_state()->RemoveSendingStream(this);
390}
391
392void AudioSendStream::SendAudioData(std::unique_ptr<AudioFrame> audio_frame) {
393 RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_);
Henrik Boströmd2c336f2019-07-03 15:11:10394 RTC_DCHECK_GT(audio_frame->sample_rate_hz_, 0);
395 double duration = static_cast<double>(audio_frame->samples_per_channel_) /
396 audio_frame->sample_rate_hz_;
397 {
398 // Note: SendAudioData() passes the frame further down the pipeline and it
399 // may eventually get sent. But this method is invoked even if we are not
400 // connected, as long as we have an AudioSendStream (created as a result of
401 // an O/A exchange). This means that we are calculating audio levels whether
402 // or not we are sending samples.
403 // TODO(https://crbug.com/webrtc/10771): All "media-source" related stats
404 // should move from send-streams to the local audio sources or tracks; a
405 // send-stream should not be required to read the microphone audio levels.
Markus Handell62872802020-07-06 13:15:07406 MutexLock lock(&audio_level_lock_);
Henrik Boströmd2c336f2019-07-03 15:11:10407 audio_level_.ComputeLevel(*audio_frame, duration);
408 }
Niels Möllerdced9f62018-11-19 09:27:07409 channel_send_->ProcessAndEncodeAudio(std::move(audio_frame));
solenberg3a941542015-11-16 15:34:50410}
411
solenbergffbbcac2016-11-17 13:25:37412bool AudioSendStream::SendTelephoneEvent(int payload_type,
Yves Gerey665174f2018-06-19 13:03:05413 int payload_frequency,
414 int event,
solenberg8842c3e2016-03-11 11:06:41415 int duration_ms) {
Artem Titova2088612021-02-03 12:33:28416 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller8fb1a6a2019-03-05 13:29:42417 channel_send_->SetSendTelephoneEventPayloadType(payload_type,
418 payload_frequency);
419 return channel_send_->SendTelephoneEventOutband(event, duration_ms);
Fredrik Solenbergb5727682015-12-04 14:22:19420}
421
solenberg94218532016-06-16 17:53:22422void AudioSendStream::SetMuted(bool muted) {
Artem Titova2088612021-02-03 12:33:28423 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möllerdced9f62018-11-19 09:27:07424 channel_send_->SetInputMute(muted);
solenberg94218532016-06-16 17:53:22425}
426
solenbergc7a8b082015-10-16 21:35:07427webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const {
Ivo Creusen56d460902017-11-24 16:29:59428 return GetStats(true);
429}
430
431webrtc::AudioSendStream::Stats AudioSendStream::GetStats(
432 bool has_remote_tracks) const {
Artem Titova2088612021-02-03 12:33:28433 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
solenberg85a04962015-10-27 10:35:21434 webrtc::AudioSendStream::Stats stats;
435 stats.local_ssrc = config_.rtp.ssrc;
Jakob Ivarssonbf087452021-11-11 12:43:49436 stats.target_bitrate_bps = channel_send_->GetTargetBitrate();
solenberg85a04962015-10-27 10:35:21437
Niels Möllerdced9f62018-11-19 09:27:07438 webrtc::CallSendStatistics call_stats = channel_send_->GetRTCPStatistics();
Niels Möllerac0a4cb2019-10-09 13:01:33439 stats.payload_bytes_sent = call_stats.payload_bytes_sent;
440 stats.header_and_padding_bytes_sent =
441 call_stats.header_and_padding_bytes_sent;
Henrik Boströmcf96e0f2019-04-17 11:51:53442 stats.retransmitted_bytes_sent = call_stats.retransmitted_bytes_sent;
solenberg85a04962015-10-27 10:35:21443 stats.packets_sent = call_stats.packetsSent;
Henrik Boströmcf96e0f2019-04-17 11:51:53444 stats.retransmitted_packets_sent = call_stats.retransmitted_packets_sent;
solenberg8b85de22015-11-16 17:48:04445 // RTT isn't known until a RTCP report is received. Until then, VoiceEngine
446 // returns 0 to indicate an error value.
447 if (call_stats.rttMs > 0) {
448 stats.rtt_ms = call_stats.rttMs;
449 }
ossu20a4b3f2017-04-27 09:08:52450 if (config_.send_codec_spec) {
451 const auto& spec = *config_.send_codec_spec;
452 stats.codec_name = spec.format.name;
Oskar Sundbom2707fb22017-11-16 09:57:35453 stats.codec_payload_type = spec.payload_type;
solenberg85a04962015-10-27 10:35:21454
455 // Get data from the last remote RTCP report.
Niels Möllerdced9f62018-11-19 09:27:07456 for (const auto& block : channel_send_->GetRemoteRTCPReportBlocks()) {
solenberg8b85de22015-11-16 17:48:04457 // Lookup report for send ssrc only.
458 if (block.source_SSRC == stats.local_ssrc) {
459 stats.packets_lost = block.cumulative_num_packets_lost;
460 stats.fraction_lost = Q8ToFloat(block.fraction_lost);
ossu20a4b3f2017-04-27 09:08:52461 // Convert timestamps to milliseconds.
462 if (spec.format.clockrate_hz / 1000 > 0) {
solenberg8b85de22015-11-16 17:48:04463 stats.jitter_ms =
ossu20a4b3f2017-04-27 09:08:52464 block.interarrival_jitter / (spec.format.clockrate_hz / 1000);
solenberg85a04962015-10-27 10:35:21465 }
solenberg8b85de22015-11-16 17:48:04466 break;
solenberg85a04962015-10-27 10:35:21467 }
468 }
469 }
470
Henrik Boströmd2c336f2019-07-03 15:11:10471 {
Markus Handell62872802020-07-06 13:15:07472 MutexLock lock(&audio_level_lock_);
Henrik Boströmd2c336f2019-07-03 15:11:10473 stats.audio_level = audio_level_.LevelFullRange();
474 stats.total_input_energy = audio_level_.TotalEnergy();
475 stats.total_input_duration = audio_level_.TotalDuration();
476 }
solenberg796b8f92017-03-02 01:02:23477
Fredrik Solenberg2a877972017-12-15 15:42:15478 stats.typing_noise_detected = audio_state()->typing_noise_detected();
Niels Möllerdced9f62018-11-19 09:27:07479 stats.ana_statistics = channel_send_->GetANAStatistics();
Per Åhgrencc73ed32020-04-26 21:56:17480
481 AudioProcessing* ap = audio_state_->audio_processing();
482 if (ap) {
483 stats.apm_statistics = ap->GetStatistics(has_remote_tracks);
484 }
solenberg85a04962015-10-27 10:35:21485
Henrik Boström6e436d12019-05-27 10:19:33486 stats.report_block_datas = std::move(call_stats.report_block_datas);
487
Jakob Ivarssone91c9922021-07-06 07:55:43488 stats.nacks_rcvd = call_stats.nacks_rcvd;
489
solenberg85a04962015-10-27 10:35:21490 return stats;
491}
492
Niels Möller8fb1a6a2019-03-05 13:29:42493void AudioSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
Erik Språng2b4d2f32020-06-29 14:37:44494 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller8fb1a6a2019-03-05 13:29:42495 channel_send_->ReceivedRTCPPacket(packet, length);
Artem Titova2088612021-02-03 12:33:28496
497 {
Erik Språng04e1bab2020-05-07 16:18:32498 // Poll if overhead has changed, which it can do if ack triggers us to stop
499 // sending mid/rid.
Markus Handell62872802020-07-06 13:15:07500 MutexLock lock(&overhead_per_packet_lock_);
Erik Språng04e1bab2020-05-07 16:18:32501 UpdateOverheadForEncoder();
Artem Titova2088612021-02-03 12:33:28502 }
503 UpdateCachedTargetAudioBitrateConstraints();
pbos1ba8d392016-05-02 03:18:34504}
505
Sebastian Janssonc0e4d452018-10-25 13:08:32506uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) {
Markus Handell3907e7b2021-06-01 07:07:20507 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Erik Språng04e1bab2020-05-07 16:18:32508
Daniel Lee93562522019-05-03 12:40:13509 // Pick a target bitrate between the constraints. Overrules the allocator if
510 // it 1) allocated a bitrate of zero to disable the stream or 2) allocated a
511 // higher than max to allow for e.g. extra FEC.
Artem Titova2088612021-02-03 12:33:28512 RTC_DCHECK(cached_constraints_.has_value());
513 update.target_bitrate.Clamp(cached_constraints_->min,
514 cached_constraints_->max);
515 update.stable_target_bitrate.Clamp(cached_constraints_->min,
516 cached_constraints_->max);
mflodman86cc6ff2016-07-26 11:44:06517
Sebastian Jansson254d8692018-11-21 18:19:00518 channel_send_->OnBitrateAllocation(update);
mflodman86cc6ff2016-07-26 11:44:06519
520 // The amount of audio protection is not exposed by the encoder, hence
521 // always returning 0.
522 return 0;
523}
524
Anton Sukhanov626015d2019-02-04 23:16:06525void AudioSendStream::SetTransportOverhead(
526 int transport_overhead_per_packet_bytes) {
Artem Titova2088612021-02-03 12:33:28527 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
528 {
529 MutexLock lock(&overhead_per_packet_lock_);
530 transport_overhead_per_packet_bytes_ = transport_overhead_per_packet_bytes;
531 UpdateOverheadForEncoder();
532 }
533 UpdateCachedTargetAudioBitrateConstraints();
Anton Sukhanov626015d2019-02-04 23:16:06534}
535
Anton Sukhanov626015d2019-02-04 23:16:06536void AudioSendStream::UpdateOverheadForEncoder() {
Artem Titova2088612021-02-03 12:33:28537 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Erik Språngcf6544a2020-05-13 12:43:11538 size_t overhead_per_packet_bytes = GetPerPacketOverheadBytes();
539 if (overhead_per_packet_ == overhead_per_packet_bytes) {
540 return;
Bjorn A Mellem413ccc42019-04-26 22:41:05541 }
Erik Språngcf6544a2020-05-13 12:43:11542 overhead_per_packet_ = overhead_per_packet_bytes;
543
Sebastian Jansson14a7cf92019-02-13 14:11:42544 channel_send_->CallEncoder([&](AudioEncoder* encoder) {
545 encoder->OnReceivedOverhead(overhead_per_packet_bytes);
Anton Sukhanov626015d2019-02-04 23:16:06546 });
Artem Titova2088612021-02-03 12:33:28547 if (total_packet_overhead_bytes_ != overhead_per_packet_bytes) {
548 total_packet_overhead_bytes_ = overhead_per_packet_bytes;
549 if (registered_with_allocator_) {
550 ConfigureBitrateObserver();
Sebastian Jansson8672cac2019-03-01 14:57:55551 }
Erik Språng04e1bab2020-05-07 16:18:32552 }
Anton Sukhanov626015d2019-02-04 23:16:06553}
554
555size_t AudioSendStream::TestOnlyGetPerPacketOverheadBytes() const {
Markus Handell62872802020-07-06 13:15:07556 MutexLock lock(&overhead_per_packet_lock_);
Anton Sukhanov626015d2019-02-04 23:16:06557 return GetPerPacketOverheadBytes();
558}
559
560size_t AudioSendStream::GetPerPacketOverheadBytes() const {
561 return transport_overhead_per_packet_bytes_ +
Erik Språng04e1bab2020-05-07 16:18:32562 rtp_rtcp_module_->ExpectedPerPacketOverhead();
michaelt79e05882016-11-08 10:50:09563}
564
ossuc3d4b482017-05-23 13:07:11565RtpState AudioSendStream::GetRtpState() const {
566 return rtp_rtcp_module_->GetRtpState();
567}
568
Niels Möllerdced9f62018-11-19 09:27:07569const voe::ChannelSendInterface* AudioSendStream::GetChannel() const {
570 return channel_send_.get();
Fredrik Solenberg8f5787a2018-01-11 12:52:30571}
572
Fredrik Solenberg2a877972017-12-15 15:42:15573internal::AudioState* AudioSendStream::audio_state() {
574 internal::AudioState* audio_state =
575 static_cast<internal::AudioState*>(audio_state_.get());
576 RTC_DCHECK(audio_state);
577 return audio_state;
578}
579
580const internal::AudioState* AudioSendStream::audio_state() const {
581 internal::AudioState* audio_state =
582 static_cast<internal::AudioState*>(audio_state_.get());
583 RTC_DCHECK(audio_state);
584 return audio_state;
585}
586
Fredrik Solenberg2a877972017-12-15 15:42:15587void AudioSendStream::StoreEncoderProperties(int sample_rate_hz,
588 size_t num_channels) {
Fredrik Solenberg2a877972017-12-15 15:42:15589 encoder_sample_rate_hz_ = sample_rate_hz;
590 encoder_num_channels_ = num_channels;
591 if (sending_) {
592 // Update AudioState's information about the stream.
593 audio_state()->AddSendingStream(this, sample_rate_hz, num_channels);
594 }
595}
596
minyue7a973442016-10-20 10:27:12597// Apply current codec settings to a single voe::Channel used for sending.
Sebastian Jansson35cf9e72019-10-04 07:30:32598bool AudioSendStream::SetupSendCodec(const Config& new_config) {
ossu20a4b3f2017-04-27 09:08:52599 RTC_DCHECK(new_config.send_codec_spec);
600 const auto& spec = *new_config.send_codec_spec;
minyue48368ad2017-05-10 11:06:11601
602 RTC_DCHECK(new_config.encoder_factory);
ossu20a4b3f2017-04-27 09:08:52603 std::unique_ptr<AudioEncoder> encoder =
Karl Wiberg77490b92018-03-21 14:18:42604 new_config.encoder_factory->MakeAudioEncoder(
605 spec.payload_type, spec.format, new_config.codec_pair_id);
minyue7a973442016-10-20 10:27:12606
ossu20a4b3f2017-04-27 09:08:52607 if (!encoder) {
Jonas Olssonabbe8412018-04-03 11:40:05608 RTC_DLOG(LS_ERROR) << "Unable to create encoder for "
609 << rtc::ToString(spec.format);
ossu20a4b3f2017-04-27 09:08:52610 return false;
611 }
Alex Narestbbbe4e12018-07-13 08:32:58612
ossu20a4b3f2017-04-27 09:08:52613 // If a bitrate has been specified for the codec, use it over the
614 // codec's default.
Christoffer Rodbro110c64b2019-03-06 08:51:08615 if (spec.target_bitrate_bps) {
ossu20a4b3f2017-04-27 09:08:52616 encoder->OnReceivedTargetAudioBitrate(*spec.target_bitrate_bps);
minyue7a973442016-10-20 10:27:12617 }
618
ossu20a4b3f2017-04-27 09:08:52619 // Enable ANA if configured (currently only used by Opus).
Mirko Bonadei43564902020-01-29 15:29:36620 if (new_config.audio_network_adaptor_config) {
ossu20a4b3f2017-04-27 09:08:52621 if (encoder->EnableAudioNetworkAdaptor(
Sebastian Jansson35cf9e72019-10-04 07:30:32622 *new_config.audio_network_adaptor_config, event_log_)) {
Jakob Ivarssoned971162020-08-11 12:05:07623 RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC "
624 << new_config.rtp.ssrc;
ossu20a4b3f2017-04-27 09:08:52625 } else {
Jakob Ivarssoned971162020-08-11 12:05:07626 RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC "
627 << new_config.rtp.ssrc;
minyue6b825df2016-10-31 11:08:32628 }
minyue7a973442016-10-20 10:27:12629 }
630
Philipp Hancke1a497562020-05-26 17:12:31631 // Wrap the encoder in an AudioEncoderCNG, if VAD is enabled.
ossu20a4b3f2017-04-27 09:08:52632 if (spec.cng_payload_type) {
Karl Wiberg23659362018-11-01 10:13:44633 AudioEncoderCngConfig cng_config;
ossu20a4b3f2017-04-27 09:08:52634 cng_config.num_channels = encoder->NumChannels();
635 cng_config.payload_type = *spec.cng_payload_type;
636 cng_config.speech_encoder = std::move(encoder);
637 cng_config.vad_mode = Vad::kVadNormal;
Karl Wiberg23659362018-11-01 10:13:44638 encoder = CreateComfortNoiseEncoder(std::move(cng_config));
ossu3b9ff382017-04-27 15:03:42639
Sebastian Jansson35cf9e72019-10-04 07:30:32640 RegisterCngPayloadType(*spec.cng_payload_type,
641 new_config.send_codec_spec->format.clockrate_hz);
minyue7a973442016-10-20 10:27:12642 }
ossu20a4b3f2017-04-27 09:08:52643
Philipp Hanckeedcd9662020-06-24 10:52:42644 // Wrap the encoder in a RED encoder, if RED is enabled.
645 if (spec.red_payload_type) {
646 AudioEncoderCopyRed::Config red_config;
647 red_config.payload_type = *spec.red_payload_type;
648 red_config.speech_encoder = std::move(encoder);
Jonas Orelanda943e732022-03-16 12:50:58649 encoder = std::make_unique<AudioEncoderCopyRed>(std::move(red_config),
650 field_trials_);
Philipp Hanckeedcd9662020-06-24 10:52:42651 }
652
Anton Sukhanov626015d2019-02-04 23:16:06653 // Set currently known overhead (used in ANA, opus only).
654 // If overhead changes later, it will be updated in UpdateOverheadForEncoder.
655 {
Markus Handell62872802020-07-06 13:15:07656 MutexLock lock(&overhead_per_packet_lock_);
Erik Språng04e1bab2020-05-07 16:18:32657 size_t overhead = GetPerPacketOverheadBytes();
658 if (overhead > 0) {
659 encoder->OnReceivedOverhead(overhead);
Bjorn A Mellem413ccc42019-04-26 22:41:05660 }
Anton Sukhanov626015d2019-02-04 23:16:06661 }
662
Sebastian Jansson35cf9e72019-10-04 07:30:32663 StoreEncoderProperties(encoder->SampleRateHz(), encoder->NumChannels());
664 channel_send_->SetEncoder(new_config.send_codec_spec->payload_type,
665 std::move(encoder));
Anton Sukhanov626015d2019-02-04 23:16:06666
minyue7a973442016-10-20 10:27:12667 return true;
668}
669
Sebastian Jansson35cf9e72019-10-04 07:30:32670bool AudioSendStream::ReconfigureSendCodec(const Config& new_config) {
671 const auto& old_config = config_;
minyue-webrtc8de18262017-07-26 12:18:40672
673 if (!new_config.send_codec_spec) {
674 // We cannot de-configure a send codec. So we will do nothing.
675 // By design, the send codec should have not been configured.
676 RTC_DCHECK(!old_config.send_codec_spec);
677 return true;
678 }
679
680 if (new_config.send_codec_spec == old_config.send_codec_spec &&
681 new_config.audio_network_adaptor_config ==
682 old_config.audio_network_adaptor_config) {
ossu20a4b3f2017-04-27 09:08:52683 return true;
684 }
685
686 // If we have no encoder, or the format or payload type's changed, create a
687 // new encoder.
688 if (!old_config.send_codec_spec ||
689 new_config.send_codec_spec->format !=
690 old_config.send_codec_spec->format ||
691 new_config.send_codec_spec->payload_type !=
Philipp Hancke6144b842021-06-04 11:49:27692 old_config.send_codec_spec->payload_type ||
693 new_config.send_codec_spec->red_payload_type !=
694 old_config.send_codec_spec->red_payload_type) {
Sebastian Jansson35cf9e72019-10-04 07:30:32695 return SetupSendCodec(new_config);
ossu20a4b3f2017-04-27 09:08:52696 }
697
Danil Chapovalovb9b146c2018-06-15 10:28:07698 const absl::optional<int>& new_target_bitrate_bps =
ossu20a4b3f2017-04-27 09:08:52699 new_config.send_codec_spec->target_bitrate_bps;
700 // If a bitrate has been specified for the codec, use it over the
701 // codec's default.
Christoffer Rodbro110c64b2019-03-06 08:51:08702 if (new_target_bitrate_bps &&
ossu20a4b3f2017-04-27 09:08:52703 new_target_bitrate_bps !=
704 old_config.send_codec_spec->target_bitrate_bps) {
Sebastian Jansson35cf9e72019-10-04 07:30:32705 channel_send_->CallEncoder([&](AudioEncoder* encoder) {
ossu20a4b3f2017-04-27 09:08:52706 encoder->OnReceivedTargetAudioBitrate(*new_target_bitrate_bps);
707 });
708 }
709
Sebastian Jansson35cf9e72019-10-04 07:30:32710 ReconfigureANA(new_config);
711 ReconfigureCNG(new_config);
ossu20a4b3f2017-04-27 09:08:52712
713 return true;
714}
715
Sebastian Jansson35cf9e72019-10-04 07:30:32716void AudioSendStream::ReconfigureANA(const Config& new_config) {
ossu20a4b3f2017-04-27 09:08:52717 if (new_config.audio_network_adaptor_config ==
Sebastian Jansson35cf9e72019-10-04 07:30:32718 config_.audio_network_adaptor_config) {
ossu20a4b3f2017-04-27 09:08:52719 return;
720 }
Mirko Bonadei43564902020-01-29 15:29:36721 if (new_config.audio_network_adaptor_config) {
Jakob Ivarssonfde2b242020-08-20 14:48:49722 // This lock needs to be acquired before CallEncoder, since it aquires
723 // another lock and we need to maintain the same order at all call sites to
724 // avoid deadlock.
725 MutexLock lock(&overhead_per_packet_lock_);
726 size_t overhead = GetPerPacketOverheadBytes();
Sebastian Jansson35cf9e72019-10-04 07:30:32727 channel_send_->CallEncoder([&](AudioEncoder* encoder) {
ossu20a4b3f2017-04-27 09:08:52728 if (encoder->EnableAudioNetworkAdaptor(
Sebastian Jansson35cf9e72019-10-04 07:30:32729 *new_config.audio_network_adaptor_config, event_log_)) {
Jakob Ivarssoned971162020-08-11 12:05:07730 RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC "
731 << new_config.rtp.ssrc;
Jakob Ivarssonfde2b242020-08-20 14:48:49732 if (overhead > 0) {
733 encoder->OnReceivedOverhead(overhead);
734 }
ossu20a4b3f2017-04-27 09:08:52735 } else {
Jakob Ivarssoned971162020-08-11 12:05:07736 RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC "
737 << new_config.rtp.ssrc;
ossu20a4b3f2017-04-27 09:08:52738 }
739 });
740 } else {
Sebastian Jansson35cf9e72019-10-04 07:30:32741 channel_send_->CallEncoder(
Sebastian Jansson14a7cf92019-02-13 14:11:42742 [&](AudioEncoder* encoder) { encoder->DisableAudioNetworkAdaptor(); });
Jakob Ivarssoned971162020-08-11 12:05:07743 RTC_LOG(LS_INFO) << "Audio network adaptor disabled on SSRC "
744 << new_config.rtp.ssrc;
ossu20a4b3f2017-04-27 09:08:52745 }
746}
747
Sebastian Jansson35cf9e72019-10-04 07:30:32748void AudioSendStream::ReconfigureCNG(const Config& new_config) {
ossu20a4b3f2017-04-27 09:08:52749 if (new_config.send_codec_spec->cng_payload_type ==
Sebastian Jansson35cf9e72019-10-04 07:30:32750 config_.send_codec_spec->cng_payload_type) {
ossu20a4b3f2017-04-27 09:08:52751 return;
752 }
753
ossu3b9ff382017-04-27 15:03:42754 // Register the CNG payload type if it's been added, don't do anything if CNG
755 // is removed. Payload types must not be redefined.
756 if (new_config.send_codec_spec->cng_payload_type) {
Sebastian Jansson35cf9e72019-10-04 07:30:32757 RegisterCngPayloadType(*new_config.send_codec_spec->cng_payload_type,
758 new_config.send_codec_spec->format.clockrate_hz);
ossu3b9ff382017-04-27 15:03:42759 }
760
ossu20a4b3f2017-04-27 09:08:52761 // Wrap or unwrap the encoder in an AudioEncoderCNG.
Sebastian Jansson35cf9e72019-10-04 07:30:32762 channel_send_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder_ptr) {
763 std::unique_ptr<AudioEncoder> old_encoder(std::move(*encoder_ptr));
764 auto sub_encoders = old_encoder->ReclaimContainedEncoders();
765 if (!sub_encoders.empty()) {
766 // Replace enc with its sub encoder. We need to put the sub
767 // encoder in a temporary first, since otherwise the old value
768 // of enc would be destroyed before the new value got assigned,
769 // which would be bad since the new value is a part of the old
770 // value.
771 auto tmp = std::move(sub_encoders[0]);
772 old_encoder = std::move(tmp);
773 }
774 if (new_config.send_codec_spec->cng_payload_type) {
775 AudioEncoderCngConfig config;
776 config.speech_encoder = std::move(old_encoder);
777 config.num_channels = config.speech_encoder->NumChannels();
778 config.payload_type = *new_config.send_codec_spec->cng_payload_type;
779 config.vad_mode = Vad::kVadNormal;
780 *encoder_ptr = CreateComfortNoiseEncoder(std::move(config));
781 } else {
782 *encoder_ptr = std::move(old_encoder);
783 }
784 });
ossu20a4b3f2017-04-27 09:08:52785}
786
787void AudioSendStream::ReconfigureBitrateObserver(
ossu20a4b3f2017-04-27 09:08:52788 const webrtc::AudioSendStream::Config& new_config) {
789 // Since the Config's default is for both of these to be -1, this test will
790 // allow us to configure the bitrate observer if the new config has bitrate
791 // limits set, but would only have us call RemoveBitrateObserver if we were
792 // previously configured with bitrate limits.
Sebastian Jansson35cf9e72019-10-04 07:30:32793 if (config_.min_bitrate_bps == new_config.min_bitrate_bps &&
794 config_.max_bitrate_bps == new_config.max_bitrate_bps &&
795 config_.bitrate_priority == new_config.bitrate_priority &&
Jakob Ivarsson47a03e82020-11-23 14:05:44796 TransportSeqNumId(config_) == TransportSeqNumId(new_config) &&
Jakob Ivarssond14525e2020-03-06 08:49:29797 config_.audio_network_adaptor_config ==
798 new_config.audio_network_adaptor_config) {
ossu20a4b3f2017-04-27 09:08:52799 return;
800 }
801
Sebastian Janssonf23131f2019-10-03 08:03:55802 if (!new_config.has_dscp && new_config.min_bitrate_bps != -1 &&
Sebastian Janssoncd0eedb2019-10-10 11:52:26803 new_config.max_bitrate_bps != -1 && TransportSeqNumId(new_config) != 0) {
Sebastian Jansson35cf9e72019-10-04 07:30:32804 rtp_transport_->AccountForAudioPacketsInPacedSender(true);
Sebastian Janssonc3eb9fd2020-01-29 16:42:52805 if (send_side_bwe_with_overhead_)
806 rtp_transport_->IncludeOverheadInPacedSender();
Artem Titova2088612021-02-03 12:33:28807 // We may get a callback immediately as the observer is registered, so
808 // make sure the bitrate limits in config_ are up-to-date.
809 config_.min_bitrate_bps = new_config.min_bitrate_bps;
810 config_.max_bitrate_bps = new_config.max_bitrate_bps;
Sebastian Jansson35cf9e72019-10-04 07:30:32811
Artem Titova2088612021-02-03 12:33:28812 config_.bitrate_priority = new_config.bitrate_priority;
813 ConfigureBitrateObserver();
Sebastian Jansson35cf9e72019-10-04 07:30:32814 rtp_rtcp_module_->SetAsPartOfAllocation(true);
ossu20a4b3f2017-04-27 09:08:52815 } else {
Sebastian Jansson35cf9e72019-10-04 07:30:32816 rtp_transport_->AccountForAudioPacketsInPacedSender(false);
817 RemoveBitrateObserver();
818 rtp_rtcp_module_->SetAsPartOfAllocation(false);
ossu20a4b3f2017-04-27 09:08:52819 }
820}
821
Sebastian Jansson8672cac2019-03-01 14:57:55822void AudioSendStream::ConfigureBitrateObserver() {
823 // This either updates the current observer or adds a new observer.
824 // TODO(srte): Add overhead compensation here.
Daniel Lee93562522019-05-03 12:40:13825 auto constraints = GetMinMaxBitrateConstraints();
Artem Titova2088612021-02-03 12:33:28826 RTC_DCHECK(constraints.has_value());
Daniel Lee93562522019-05-03 12:40:13827
Sebastian Jansson0429f782019-10-03 16:32:45828 DataRate priority_bitrate = allocation_settings_.priority_bitrate;
Sebastian Janssonf23131f2019-10-03 08:03:55829 if (send_side_bwe_with_overhead_) {
Sebastian Jansson0429f782019-10-03 16:32:45830 if (use_legacy_overhead_calculation_) {
831 // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12)
832 constexpr int kOverheadPerPacket = 20 + 8 + 10 + 12;
Danil Chapovalov0c626af2020-02-10 10:16:00833 const TimeDelta kMinPacketDuration = TimeDelta::Millis(20);
Sebastian Jansson0429f782019-10-03 16:32:45834 DataRate max_overhead =
Danil Chapovalovcad3e0e2020-02-17 17:46:07835 DataSize::Bytes(kOverheadPerPacket) / kMinPacketDuration;
Sebastian Jansson0429f782019-10-03 16:32:45836 priority_bitrate += max_overhead;
837 } else {
838 RTC_DCHECK(frame_length_range_);
Erik Språng04e1bab2020-05-07 16:18:32839 const DataSize overhead_per_packet =
Danil Chapovalovcad3e0e2020-02-17 17:46:07840 DataSize::Bytes(total_packet_overhead_bytes_);
Erik Språng04e1bab2020-05-07 16:18:32841 DataRate min_overhead = overhead_per_packet / frame_length_range_->second;
Jakob Ivarsson01ab0842020-03-06 08:59:56842 priority_bitrate += min_overhead;
Sebastian Jansson0429f782019-10-03 16:32:45843 }
Sebastian Janssonf23131f2019-10-03 08:03:55844 }
Sebastian Janssonf23131f2019-10-03 08:03:55845 if (allocation_settings_.priority_bitrate_raw)
846 priority_bitrate = *allocation_settings_.priority_bitrate_raw;
847
Markus Handell3907e7b2021-06-01 07:07:20848 rtp_transport_queue_->PostTask([this, constraints, priority_bitrate,
849 config_bitrate_priority =
850 config_.bitrate_priority] {
851 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Artem Titova2088612021-02-03 12:33:28852 bitrate_allocator_->AddObserver(
853 this,
854 MediaStreamAllocationConfig{
855 constraints->min.bps<uint32_t>(), constraints->max.bps<uint32_t>(),
856 0, priority_bitrate.bps(), true,
857 allocation_settings_.bitrate_priority.value_or(
858 config_bitrate_priority)});
859 });
Jakob Ivarssond14525e2020-03-06 08:49:29860 registered_with_allocator_ = true;
ossu20a4b3f2017-04-27 09:08:52861}
862
863void AudioSendStream::RemoveBitrateObserver() {
Artem Titova2088612021-02-03 12:33:28864 registered_with_allocator_ = false;
Niels Möllerc572ff32018-11-07 07:43:50865 rtc::Event thread_sync_event;
Markus Handell3907e7b2021-06-01 07:07:20866 rtp_transport_queue_->PostTask([this, &thread_sync_event] {
867 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
ossu20a4b3f2017-04-27 09:08:52868 bitrate_allocator_->RemoveObserver(this);
869 thread_sync_event.Set();
870 });
871 thread_sync_event.Wait(rtc::Event::kForever);
872}
873
Artem Titova2088612021-02-03 12:33:28874absl::optional<AudioSendStream::TargetAudioBitrateConstraints>
Daniel Lee93562522019-05-03 12:40:13875AudioSendStream::GetMinMaxBitrateConstraints() const {
Artem Titova2088612021-02-03 12:33:28876 if (config_.min_bitrate_bps < 0 || config_.max_bitrate_bps < 0) {
877 RTC_LOG(LS_WARNING) << "Config is invalid: min_bitrate_bps="
878 << config_.min_bitrate_bps
879 << "; max_bitrate_bps=" << config_.max_bitrate_bps
880 << "; both expected greater or equal to 0";
881 return absl::nullopt;
882 }
Daniel Lee93562522019-05-03 12:40:13883 TargetAudioBitrateConstraints constraints{
Danil Chapovalovcad3e0e2020-02-17 17:46:07884 DataRate::BitsPerSec(config_.min_bitrate_bps),
885 DataRate::BitsPerSec(config_.max_bitrate_bps)};
Daniel Lee93562522019-05-03 12:40:13886
887 // If bitrates were explicitly overriden via field trial, use those values.
Sebastian Janssonf23131f2019-10-03 08:03:55888 if (allocation_settings_.min_bitrate)
889 constraints.min = *allocation_settings_.min_bitrate;
890 if (allocation_settings_.max_bitrate)
891 constraints.max = *allocation_settings_.max_bitrate;
Daniel Lee93562522019-05-03 12:40:13892
Sebastian Jansson62aee932019-10-02 10:27:06893 RTC_DCHECK_GE(constraints.min, DataRate::Zero());
894 RTC_DCHECK_GE(constraints.max, DataRate::Zero());
Artem Titova2088612021-02-03 12:33:28895 if (constraints.max < constraints.min) {
896 RTC_LOG(LS_WARNING) << "TargetAudioBitrateConstraints::max is less than "
897 << "TargetAudioBitrateConstraints::min";
898 return absl::nullopt;
899 }
Sebastian Janssonf23131f2019-10-03 08:03:55900 if (send_side_bwe_with_overhead_) {
Sebastian Jansson62aee932019-10-02 10:27:06901 if (use_legacy_overhead_calculation_) {
902 // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12)
Danil Chapovalovcad3e0e2020-02-17 17:46:07903 const DataSize kOverheadPerPacket = DataSize::Bytes(20 + 8 + 10 + 12);
Sebastian Jansson62aee932019-10-02 10:27:06904 const TimeDelta kMaxFrameLength =
Danil Chapovalov0c626af2020-02-10 10:16:00905 TimeDelta::Millis(60); // Based on Opus spec
Sebastian Jansson62aee932019-10-02 10:27:06906 const DataRate kMinOverhead = kOverheadPerPacket / kMaxFrameLength;
907 constraints.min += kMinOverhead;
908 constraints.max += kMinOverhead;
909 } else {
Artem Titova2088612021-02-03 12:33:28910 if (!frame_length_range_.has_value()) {
911 RTC_LOG(LS_WARNING) << "frame_length_range_ is not set";
912 return absl::nullopt;
913 }
Sebastian Jansson62aee932019-10-02 10:27:06914 const DataSize kOverheadPerPacket =
Danil Chapovalovcad3e0e2020-02-17 17:46:07915 DataSize::Bytes(total_packet_overhead_bytes_);
Sebastian Jansson62aee932019-10-02 10:27:06916 constraints.min += kOverheadPerPacket / frame_length_range_->second;
917 constraints.max += kOverheadPerPacket / frame_length_range_->first;
918 }
Daniel Lee93562522019-05-03 12:40:13919 }
920 return constraints;
921}
922
ossu3b9ff382017-04-27 15:03:42923void AudioSendStream::RegisterCngPayloadType(int payload_type,
924 int clockrate_hz) {
Niels Mölleree5ccbc2019-03-06 15:47:29925 channel_send_->RegisterCngPayloadType(payload_type, clockrate_hz);
ossu3b9ff382017-04-27 15:03:42926}
Artem Titova2088612021-02-03 12:33:28927
928void AudioSendStream::UpdateCachedTargetAudioBitrateConstraints() {
929 absl::optional<AudioSendStream::TargetAudioBitrateConstraints>
930 new_constraints = GetMinMaxBitrateConstraints();
931 if (!new_constraints.has_value()) {
932 return;
933 }
Markus Handell3907e7b2021-06-01 07:07:20934 rtp_transport_queue_->PostTask([this, new_constraints]() {
935 RTC_DCHECK_RUN_ON(rtp_transport_queue_);
Artem Titova2088612021-02-03 12:33:28936 cached_constraints_ = new_constraints;
937 });
938}
939
solenbergc7a8b082015-10-16 21:35:07940} // namespace internal
941} // namespace webrtc