solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 11 | #include "audio/audio_send_stream.h" |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 12 | |
Mirko Bonadei | 317a1f0 | 2019-09-17 15:06:18 | [diff] [blame] | 13 | #include <memory> |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 14 | #include <string> |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 15 | #include <utility> |
| 16 | #include <vector> |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 17 | |
Yves Gerey | 988cc08 | 2018-10-23 10:03:01 | [diff] [blame] | 18 | #include "api/audio_codecs/audio_encoder.h" |
| 19 | #include "api/audio_codecs/audio_encoder_factory.h" |
| 20 | #include "api/audio_codecs/audio_format.h" |
| 21 | #include "api/call/transport.h" |
Steve Anton | 10542f2 | 2019-01-11 17:11:00 | [diff] [blame] | 22 | #include "api/crypto/frame_encryptor_interface.h" |
Artem Titov | 741daaf | 2019-03-21 13:37:36 | [diff] [blame] | 23 | #include "api/function_view.h" |
Danil Chapovalov | 83bbe91 | 2019-08-07 10:24:53 | [diff] [blame] | 24 | #include "api/rtc_event_log/rtc_event_log.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 25 | #include "audio/audio_state.h" |
Yves Gerey | 988cc08 | 2018-10-23 10:03:01 | [diff] [blame] | 26 | #include "audio/channel_send.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 27 | #include "audio/conversion.h" |
Yves Gerey | 988cc08 | 2018-10-23 10:03:01 | [diff] [blame] | 28 | #include "call/rtp_config.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 29 | #include "call/rtp_transport_controller_send_interface.h" |
Yves Gerey | 988cc08 | 2018-10-23 10:03:01 | [diff] [blame] | 30 | #include "common_audio/vad/include/vad.h" |
Oskar Sundbom | 56ef305 | 2018-10-30 15:11:02 | [diff] [blame] | 31 | #include "logging/rtc_event_log/events/rtc_event_audio_send_stream_config.h" |
Oskar Sundbom | 56ef305 | 2018-10-30 15:11:02 | [diff] [blame] | 32 | #include "logging/rtc_event_log/rtc_stream_config.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 33 | #include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" |
Philipp Hancke | edcd966 | 2020-06-24 10:52:42 | [diff] [blame] | 34 | #include "modules/audio_coding/codecs/red/audio_encoder_copy_red.h" |
Yves Gerey | 988cc08 | 2018-10-23 10:03:01 | [diff] [blame] | 35 | #include "modules/audio_processing/include/audio_processing.h" |
Sebastian Jansson | 6298b56 | 2020-01-14 16:55:19 | [diff] [blame] | 36 | #include "modules/rtp_rtcp/source/rtp_header_extensions.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 37 | #include "rtc_base/checks.h" |
| 38 | #include "rtc_base/event.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 39 | #include "rtc_base/logging.h" |
Jonas Olsson | abbe841 | 2018-04-03 11:40:05 | [diff] [blame] | 40 | #include "rtc_base/strings/audio_format_to_string.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 41 | #include "rtc_base/task_queue.h" |
Alex Narest | cedd351 | 2017-12-07 19:54:55 | [diff] [blame] | 42 | #include "system_wrappers/include/field_trial.h" |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 43 | |
| 44 | namespace webrtc { |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 45 | namespace { |
elad.alon | d12a8e1 | 2017-03-23 18:04:48 | [diff] [blame] | 46 | |
Oskar Sundbom | 56ef305 | 2018-10-30 15:11:02 | [diff] [blame] | 47 | void UpdateEventLogStreamConfig(RtcEventLog* event_log, |
| 48 | const AudioSendStream::Config& config, |
| 49 | const AudioSendStream::Config* old_config) { |
| 50 | using SendCodecSpec = AudioSendStream::Config::SendCodecSpec; |
| 51 | // Only update if any of the things we log have changed. |
| 52 | auto payload_types_equal = [](const absl::optional<SendCodecSpec>& a, |
| 53 | const absl::optional<SendCodecSpec>& b) { |
| 54 | if (a.has_value() && b.has_value()) { |
| 55 | return a->format.name == b->format.name && |
| 56 | a->payload_type == b->payload_type; |
| 57 | } |
| 58 | return !a.has_value() && !b.has_value(); |
| 59 | }; |
| 60 | |
| 61 | if (old_config && config.rtp.ssrc == old_config->rtp.ssrc && |
| 62 | config.rtp.extensions == old_config->rtp.extensions && |
| 63 | payload_types_equal(config.send_codec_spec, |
| 64 | old_config->send_codec_spec)) { |
| 65 | return; |
| 66 | } |
| 67 | |
Mirko Bonadei | 317a1f0 | 2019-09-17 15:06:18 | [diff] [blame] | 68 | auto rtclog_config = std::make_unique<rtclog::StreamConfig>(); |
Oskar Sundbom | 56ef305 | 2018-10-30 15:11:02 | [diff] [blame] | 69 | rtclog_config->local_ssrc = config.rtp.ssrc; |
| 70 | rtclog_config->rtp_extensions = config.rtp.extensions; |
| 71 | if (config.send_codec_spec) { |
| 72 | rtclog_config->codecs.emplace_back(config.send_codec_spec->format.name, |
| 73 | config.send_codec_spec->payload_type, 0); |
| 74 | } |
Mirko Bonadei | 317a1f0 | 2019-09-17 15:06:18 | [diff] [blame] | 75 | event_log->Log(std::make_unique<RtcEventAudioSendStreamConfig>( |
Oskar Sundbom | 56ef305 | 2018-10-30 15:11:02 | [diff] [blame] | 76 | std::move(rtclog_config))); |
| 77 | } |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 78 | } // namespace |
| 79 | |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 80 | constexpr char AudioAllocationConfig::kKey[]; |
| 81 | |
| 82 | std::unique_ptr<StructParametersParser> AudioAllocationConfig::Parser() { |
| 83 | return StructParametersParser::Create( // |
| 84 | "min", &min_bitrate, // |
| 85 | "max", &max_bitrate, // |
| 86 | "prio_rate", &priority_bitrate, // |
| 87 | "prio_rate_raw", &priority_bitrate_raw, // |
| 88 | "rate_prio", &bitrate_priority); |
| 89 | } |
| 90 | |
| 91 | AudioAllocationConfig::AudioAllocationConfig() { |
| 92 | Parser()->Parse(field_trial::FindFullName(kKey)); |
| 93 | if (priority_bitrate_raw && !priority_bitrate.IsZero()) { |
| 94 | RTC_LOG(LS_WARNING) << "'priority_bitrate' and '_raw' are mutually " |
| 95 | "exclusive but both were configured."; |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | namespace internal { |
solenberg | 566ef24 | 2015-11-06 23:34:49 | [diff] [blame] | 100 | AudioSendStream::AudioSendStream( |
Sebastian Jansson | 977b335 | 2019-03-04 16:43:34 | [diff] [blame] | 101 | Clock* clock, |
solenberg | 566ef24 | 2015-11-06 23:34:49 | [diff] [blame] | 102 | const webrtc::AudioSendStream::Config& config, |
Stefan Holmer | b86d4e4 | 2015-12-07 09:26:18 | [diff] [blame] | 103 | const rtc::scoped_refptr<webrtc::AudioState>& audio_state, |
Sebastian Jansson | 44dd9f2 | 2019-03-08 13:50:30 | [diff] [blame] | 104 | TaskQueueFactory* task_queue_factory, |
Niels Möller | 7d76a31 | 2018-10-26 10:57:07 | [diff] [blame] | 105 | RtpTransportControllerSendInterface* rtp_transport, |
Niels Möller | 67b011d | 2018-10-22 11:00:40 | [diff] [blame] | 106 | BitrateAllocatorInterface* bitrate_allocator, |
michaelt | 9332b7d | 2016-11-30 15:51:13 | [diff] [blame] | 107 | RtcEventLog* event_log, |
ossu | c3d4b48 | 2017-05-23 13:07:11 | [diff] [blame] | 108 | RtcpRttStats* rtcp_rtt_stats, |
Sam Zackrisson | ff05816 | 2018-11-20 16:15:13 | [diff] [blame] | 109 | const absl::optional<RtpState>& suspended_rtp_state) |
Sebastian Jansson | 977b335 | 2019-03-04 16:43:34 | [diff] [blame] | 110 | : AudioSendStream(clock, |
| 111 | config, |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 112 | audio_state, |
Sebastian Jansson | 44dd9f2 | 2019-03-08 13:50:30 | [diff] [blame] | 113 | task_queue_factory, |
Niels Möller | 7d76a31 | 2018-10-26 10:57:07 | [diff] [blame] | 114 | rtp_transport, |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 115 | bitrate_allocator, |
| 116 | event_log, |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 117 | suspended_rtp_state, |
Erik Språng | 2b4d2f3 | 2020-06-29 14:37:44 | [diff] [blame] | 118 | voe::CreateChannelSend( |
| 119 | clock, |
| 120 | task_queue_factory, |
Erik Språng | 2b4d2f3 | 2020-06-29 14:37:44 | [diff] [blame] | 121 | config.send_transport, |
| 122 | rtcp_rtt_stats, |
| 123 | event_log, |
| 124 | config.frame_encryptor, |
| 125 | config.crypto_options, |
| 126 | config.rtp.extmap_allow_mixed, |
| 127 | config.rtcp_report_interval_ms, |
| 128 | config.rtp.ssrc, |
| 129 | config.frame_transformer, |
| 130 | rtp_transport->transport_feedback_observer())) {} |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 131 | |
| 132 | AudioSendStream::AudioSendStream( |
Sebastian Jansson | 977b335 | 2019-03-04 16:43:34 | [diff] [blame] | 133 | Clock* clock, |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 134 | const webrtc::AudioSendStream::Config& config, |
| 135 | const rtc::scoped_refptr<webrtc::AudioState>& audio_state, |
Sebastian Jansson | 44dd9f2 | 2019-03-08 13:50:30 | [diff] [blame] | 136 | TaskQueueFactory* task_queue_factory, |
Niels Möller | 7d76a31 | 2018-10-26 10:57:07 | [diff] [blame] | 137 | RtpTransportControllerSendInterface* rtp_transport, |
Niels Möller | 67b011d | 2018-10-22 11:00:40 | [diff] [blame] | 138 | BitrateAllocatorInterface* bitrate_allocator, |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 139 | RtcEventLog* event_log, |
Danil Chapovalov | b9b146c | 2018-06-15 10:28:07 | [diff] [blame] | 140 | const absl::optional<RtpState>& suspended_rtp_state, |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 141 | std::unique_ptr<voe::ChannelSendInterface> channel_send) |
Sebastian Jansson | 977b335 | 2019-03-04 16:43:34 | [diff] [blame] | 142 | : clock_(clock), |
Markus Handell | 3907e7b | 2021-06-01 07:07:20 | [diff] [blame] | 143 | rtp_transport_queue_(rtp_transport->GetWorkerQueue()), |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 144 | allocate_audio_without_feedback_( |
| 145 | field_trial::IsEnabled("WebRTC-Audio-ABWENoTWCC")), |
| 146 | enable_audio_alr_probing_( |
| 147 | !field_trial::IsDisabled("WebRTC-Audio-AlrProbing")), |
| 148 | send_side_bwe_with_overhead_( |
Jakob Ivarsson | 36274f9 | 2020-10-22 11:01:07 | [diff] [blame] | 149 | !field_trial::IsDisabled("WebRTC-SendSideBwe-WithOverhead")), |
Bjorn A Mellem | 7a9a092 | 2019-11-26 17:19:40 | [diff] [blame] | 150 | config_(Config(/*send_transport=*/nullptr)), |
mflodman | 86cc6ff | 2016-07-26 11:44:06 | [diff] [blame] | 151 | audio_state_(audio_state), |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 152 | channel_send_(std::move(channel_send)), |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 153 | event_log_(event_log), |
Sebastian Jansson | 62aee93 | 2019-10-02 10:27:06 | [diff] [blame] | 154 | use_legacy_overhead_calculation_( |
Sebastian Jansson | bef818d | 2020-01-30 13:09:48 | [diff] [blame] | 155 | field_trial::IsEnabled("WebRTC-Audio-LegacyOverhead")), |
michaelt | f4caaab | 2017-01-17 07:55:07 | [diff] [blame] | 156 | bitrate_allocator_(bitrate_allocator), |
Niels Möller | 7d76a31 | 2018-10-26 10:57:07 | [diff] [blame] | 157 | rtp_transport_(rtp_transport), |
Sebastian Jansson | 6298b56 | 2020-01-14 16:55:19 | [diff] [blame] | 158 | rtp_rtcp_module_(channel_send_->GetRtpRtcp()), |
Sam Zackrisson | ff05816 | 2018-11-20 16:15:13 | [diff] [blame] | 159 | suspended_rtp_state_(suspended_rtp_state) { |
Jonas Olsson | 24ea822 | 2018-01-25 09:14:29 | [diff] [blame] | 160 | RTC_LOG(LS_INFO) << "AudioSendStream: " << config.rtp.ssrc; |
Markus Handell | 3907e7b | 2021-06-01 07:07:20 | [diff] [blame] | 161 | RTC_DCHECK(rtp_transport_queue_); |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 162 | RTC_DCHECK(audio_state_); |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 163 | RTC_DCHECK(channel_send_); |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 164 | RTC_DCHECK(bitrate_allocator_); |
Sebastian Jansson | 0b69826 | 2019-03-07 08:17:19 | [diff] [blame] | 165 | RTC_DCHECK(rtp_transport); |
| 166 | |
ossu | c3d4b48 | 2017-05-23 13:07:11 | [diff] [blame] | 167 | RTC_DCHECK(rtp_rtcp_module_); |
mflodman | 3d7db26 | 2016-04-29 07:57:13 | [diff] [blame] | 168 | |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 169 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 170 | ConfigureStream(config, true); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 171 | UpdateCachedTargetAudioBitrateConstraints(); |
Sebastian Jansson | c01367d | 2019-04-08 13:20:44 | [diff] [blame] | 172 | pacer_thread_checker_.Detach(); |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 173 | } |
| 174 | |
| 175 | AudioSendStream::~AudioSendStream() { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 176 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Jonas Olsson | 24ea822 | 2018-01-25 09:14:29 | [diff] [blame] | 177 | RTC_LOG(LS_INFO) << "~AudioSendStream: " << config_.rtp.ssrc; |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 178 | RTC_DCHECK(!sending_); |
Sebastian Jansson | 0a6510d | 2019-10-04 07:31:08 | [diff] [blame] | 179 | channel_send_->ResetSenderCongestionControlObjects(); |
Sebastian Jansson | 8672cac | 2019-03-01 14:57:55 | [diff] [blame] | 180 | // Blocking call to synchronize state with worker queue to ensure that there |
| 181 | // are no pending tasks left that keeps references to audio. |
| 182 | rtc::Event thread_sync_event; |
Markus Handell | 3907e7b | 2021-06-01 07:07:20 | [diff] [blame] | 183 | rtp_transport_queue_->PostTask([&] { thread_sync_event.Set(); }); |
Sebastian Jansson | 8672cac | 2019-03-01 14:57:55 | [diff] [blame] | 184 | thread_sync_event.Wait(rtc::Event::kForever); |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 185 | } |
| 186 | |
eladalon | abbc430 | 2017-07-26 09:09:44 | [diff] [blame] | 187 | const webrtc::AudioSendStream::Config& AudioSendStream::GetConfig() const { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 188 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
eladalon | abbc430 | 2017-07-26 09:09:44 | [diff] [blame] | 189 | return config_; |
| 190 | } |
| 191 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 192 | void AudioSendStream::Reconfigure( |
| 193 | const webrtc::AudioSendStream::Config& new_config) { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 194 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 195 | ConfigureStream(new_config, false); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 196 | } |
| 197 | |
Alex Narest | cedd351 | 2017-12-07 19:54:55 | [diff] [blame] | 198 | AudioSendStream::ExtensionIds AudioSendStream::FindExtensionIds( |
| 199 | const std::vector<RtpExtension>& extensions) { |
| 200 | ExtensionIds ids; |
| 201 | for (const auto& extension : extensions) { |
| 202 | if (extension.uri == RtpExtension::kAudioLevelUri) { |
| 203 | ids.audio_level = extension.id; |
Sebastian Jansson | 71c6b56 | 2019-08-14 09:31:02 | [diff] [blame] | 204 | } else if (extension.uri == RtpExtension::kAbsSendTimeUri) { |
| 205 | ids.abs_send_time = extension.id; |
Alex Narest | cedd351 | 2017-12-07 19:54:55 | [diff] [blame] | 206 | } else if (extension.uri == RtpExtension::kTransportSequenceNumberUri) { |
| 207 | ids.transport_sequence_number = extension.id; |
Steve Anton | bb50ce5 | 2018-03-26 17:24:32 | [diff] [blame] | 208 | } else if (extension.uri == RtpExtension::kMidUri) { |
| 209 | ids.mid = extension.id; |
Amit Hilbuch | 77938e6 | 2018-12-21 17:23:38 | [diff] [blame] | 210 | } else if (extension.uri == RtpExtension::kRidUri) { |
| 211 | ids.rid = extension.id; |
| 212 | } else if (extension.uri == RtpExtension::kRepairedRidUri) { |
| 213 | ids.repaired_rid = extension.id; |
Minyue Li | 74dadc1 | 2020-03-05 10:33:13 | [diff] [blame] | 214 | } else if (extension.uri == RtpExtension::kAbsoluteCaptureTimeUri) { |
| 215 | ids.abs_capture_time = extension.id; |
Alex Narest | cedd351 | 2017-12-07 19:54:55 | [diff] [blame] | 216 | } |
| 217 | } |
| 218 | return ids; |
| 219 | } |
| 220 | |
Sebastian Jansson | 470a5ea | 2019-01-23 11:37:49 | [diff] [blame] | 221 | int AudioSendStream::TransportSeqNumId(const AudioSendStream::Config& config) { |
| 222 | return FindExtensionIds(config.rtp.extensions).transport_sequence_number; |
| 223 | } |
| 224 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 225 | void AudioSendStream::ConfigureStream( |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 226 | const webrtc::AudioSendStream::Config& new_config, |
| 227 | bool first_time) { |
Jonas Olsson | 24ea822 | 2018-01-25 09:14:29 | [diff] [blame] | 228 | RTC_LOG(LS_INFO) << "AudioSendStream::ConfigureStream: " |
| 229 | << new_config.ToString(); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 230 | UpdateEventLogStreamConfig(event_log_, new_config, |
| 231 | first_time ? nullptr : &config_); |
Oskar Sundbom | 56ef305 | 2018-10-30 15:11:02 | [diff] [blame] | 232 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 233 | const auto& old_config = config_; |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 234 | |
Niels Möller | e977199 | 2018-11-26 09:55:07 | [diff] [blame] | 235 | // Configuration parameters which cannot be changed. |
| 236 | RTC_DCHECK(first_time || |
| 237 | old_config.send_transport == new_config.send_transport); |
Erik Språng | 70efdde | 2019-08-21 11:36:20 | [diff] [blame] | 238 | RTC_DCHECK(first_time || old_config.rtp.ssrc == new_config.rtp.ssrc); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 239 | if (suspended_rtp_state_ && first_time) { |
| 240 | rtp_rtcp_module_->SetRtpState(*suspended_rtp_state_); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 241 | } |
| 242 | if (first_time || old_config.rtp.c_name != new_config.rtp.c_name) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 243 | channel_send_->SetRTCP_CNAME(new_config.rtp.c_name); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 244 | } |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 245 | |
Benjamin Wright | 84583f6 | 2018-10-04 21:22:34 | [diff] [blame] | 246 | // Enable the frame encryptor if a new frame encryptor has been provided. |
| 247 | if (first_time || new_config.frame_encryptor != old_config.frame_encryptor) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 248 | channel_send_->SetFrameEncryptor(new_config.frame_encryptor); |
Benjamin Wright | 84583f6 | 2018-10-04 21:22:34 | [diff] [blame] | 249 | } |
| 250 | |
Johannes Kron | 9190b82 | 2018-10-29 10:22:05 | [diff] [blame] | 251 | if (first_time || |
Marina Ciocea | d2aa8f9 | 2020-03-31 09:29:56 | [diff] [blame] | 252 | new_config.frame_transformer != old_config.frame_transformer) { |
| 253 | channel_send_->SetEncoderToPacketizerFrameTransformer( |
| 254 | new_config.frame_transformer); |
| 255 | } |
| 256 | |
| 257 | if (first_time || |
Johannes Kron | 9190b82 | 2018-10-29 10:22:05 | [diff] [blame] | 258 | new_config.rtp.extmap_allow_mixed != old_config.rtp.extmap_allow_mixed) { |
Sebastian Jansson | 6298b56 | 2020-01-14 16:55:19 | [diff] [blame] | 259 | rtp_rtcp_module_->SetExtmapAllowMixed(new_config.rtp.extmap_allow_mixed); |
Johannes Kron | 9190b82 | 2018-10-29 10:22:05 | [diff] [blame] | 260 | } |
| 261 | |
Alex Narest | cedd351 | 2017-12-07 19:54:55 | [diff] [blame] | 262 | const ExtensionIds old_ids = FindExtensionIds(old_config.rtp.extensions); |
| 263 | const ExtensionIds new_ids = FindExtensionIds(new_config.rtp.extensions); |
Yves Gerey | 1704801 | 2019-07-26 15:49:52 | [diff] [blame] | 264 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 265 | // Audio level indication |
| 266 | if (first_time || new_ids.audio_level != old_ids.audio_level) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 267 | channel_send_->SetSendAudioLevelIndicationStatus(new_ids.audio_level != 0, |
| 268 | new_ids.audio_level); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 269 | } |
Sebastian Jansson | 71c6b56 | 2019-08-14 09:31:02 | [diff] [blame] | 270 | |
| 271 | if (first_time || new_ids.abs_send_time != old_ids.abs_send_time) { |
Sebastian Jansson | 6298b56 | 2020-01-14 16:55:19 | [diff] [blame] | 272 | rtp_rtcp_module_->DeregisterSendRtpHeaderExtension( |
Sebastian Jansson | 71c6b56 | 2019-08-14 09:31:02 | [diff] [blame] | 273 | kRtpExtensionAbsoluteSendTime); |
| 274 | if (new_ids.abs_send_time) { |
Danil Chapovalov | d0321c5 | 2021-09-14 10:58:51 | [diff] [blame] | 275 | rtp_rtcp_module_->RegisterRtpHeaderExtension(AbsoluteSendTime::Uri(), |
Sebastian Jansson | f39c815 | 2019-10-14 15:32:21 | [diff] [blame] | 276 | new_ids.abs_send_time); |
Sebastian Jansson | 71c6b56 | 2019-08-14 09:31:02 | [diff] [blame] | 277 | } |
| 278 | } |
| 279 | |
Sebastian Jansson | 8d9c540 | 2017-11-15 16:22:16 | [diff] [blame] | 280 | bool transport_seq_num_id_changed = |
| 281 | new_ids.transport_sequence_number != old_ids.transport_sequence_number; |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 282 | if (first_time || |
| 283 | (transport_seq_num_id_changed && !allocate_audio_without_feedback_)) { |
ossu | 1129df2 | 2017-06-30 08:38:56 | [diff] [blame] | 284 | if (!first_time) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 285 | channel_send_->ResetSenderCongestionControlObjects(); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 286 | } |
| 287 | |
Sebastian Jansson | 8d9c540 | 2017-11-15 16:22:16 | [diff] [blame] | 288 | RtcpBandwidthObserver* bandwidth_observer = nullptr; |
Sebastian Jansson | 470a5ea | 2019-01-23 11:37:49 | [diff] [blame] | 289 | |
Jakob Ivarsson | 47a03e8 | 2020-11-23 14:05:44 | [diff] [blame] | 290 | if (!allocate_audio_without_feedback_ && |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 291 | new_ids.transport_sequence_number != 0) { |
Sebastian Jansson | 6298b56 | 2020-01-14 16:55:19 | [diff] [blame] | 292 | rtp_rtcp_module_->RegisterRtpHeaderExtension( |
Danil Chapovalov | d0321c5 | 2021-09-14 10:58:51 | [diff] [blame] | 293 | TransportSequenceNumber::Uri(), new_ids.transport_sequence_number); |
Sebastian Jansson | 8d9c540 | 2017-11-15 16:22:16 | [diff] [blame] | 294 | // Probing in application limited region is only used in combination with |
| 295 | // send side congestion control, wich depends on feedback packets which |
| 296 | // requires transport sequence numbers to be enabled. |
Sebastian Jansson | 0a6510d | 2019-10-04 07:31:08 | [diff] [blame] | 297 | // Optionally request ALR probing but do not override any existing |
| 298 | // request from other streams. |
| 299 | if (enable_audio_alr_probing_) { |
| 300 | rtp_transport_->EnablePeriodicAlrProbing(true); |
Niels Möller | 7d76a31 | 2018-10-26 10:57:07 | [diff] [blame] | 301 | } |
Sebastian Jansson | 0a6510d | 2019-10-04 07:31:08 | [diff] [blame] | 302 | bandwidth_observer = rtp_transport_->GetBandwidthObserver(); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 303 | } |
Sebastian Jansson | 0a6510d | 2019-10-04 07:31:08 | [diff] [blame] | 304 | channel_send_->RegisterSenderCongestionControlObjects(rtp_transport_, |
| 305 | bandwidth_observer); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 306 | } |
Steve Anton | bb50ce5 | 2018-03-26 17:24:32 | [diff] [blame] | 307 | // MID RTP header extension. |
Steve Anton | 003930a | 2018-03-29 19:37:21 | [diff] [blame] | 308 | if ((first_time || new_ids.mid != old_ids.mid || |
| 309 | new_config.rtp.mid != old_config.rtp.mid) && |
| 310 | new_ids.mid != 0 && !new_config.rtp.mid.empty()) { |
Danil Chapovalov | d0321c5 | 2021-09-14 10:58:51 | [diff] [blame] | 311 | rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpMid::Uri(), new_ids.mid); |
Sebastian Jansson | 6298b56 | 2020-01-14 16:55:19 | [diff] [blame] | 312 | rtp_rtcp_module_->SetMid(new_config.rtp.mid); |
Steve Anton | bb50ce5 | 2018-03-26 17:24:32 | [diff] [blame] | 313 | } |
| 314 | |
Amit Hilbuch | 77938e6 | 2018-12-21 17:23:38 | [diff] [blame] | 315 | // RID RTP header extension |
| 316 | if ((first_time || new_ids.rid != old_ids.rid || |
| 317 | new_ids.repaired_rid != old_ids.repaired_rid || |
| 318 | new_config.rtp.rid != old_config.rtp.rid)) { |
Sebastian Jansson | 6298b56 | 2020-01-14 16:55:19 | [diff] [blame] | 319 | if (new_ids.rid != 0 || new_ids.repaired_rid != 0) { |
| 320 | if (new_config.rtp.rid.empty()) { |
Danil Chapovalov | d0321c5 | 2021-09-14 10:58:51 | [diff] [blame] | 321 | rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(RtpStreamId::Uri()); |
Sebastian Jansson | 6298b56 | 2020-01-14 16:55:19 | [diff] [blame] | 322 | } else if (new_ids.repaired_rid != 0) { |
Danil Chapovalov | d0321c5 | 2021-09-14 10:58:51 | [diff] [blame] | 323 | rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpStreamId::Uri(), |
Sebastian Jansson | 6298b56 | 2020-01-14 16:55:19 | [diff] [blame] | 324 | new_ids.repaired_rid); |
| 325 | } else { |
Danil Chapovalov | d0321c5 | 2021-09-14 10:58:51 | [diff] [blame] | 326 | rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpStreamId::Uri(), |
Sebastian Jansson | 6298b56 | 2020-01-14 16:55:19 | [diff] [blame] | 327 | new_ids.rid); |
| 328 | } |
| 329 | } |
| 330 | rtp_rtcp_module_->SetRid(new_config.rtp.rid); |
Amit Hilbuch | 77938e6 | 2018-12-21 17:23:38 | [diff] [blame] | 331 | } |
| 332 | |
Minyue Li | 74dadc1 | 2020-03-05 10:33:13 | [diff] [blame] | 333 | if (first_time || new_ids.abs_capture_time != old_ids.abs_capture_time) { |
| 334 | rtp_rtcp_module_->DeregisterSendRtpHeaderExtension( |
| 335 | kRtpExtensionAbsoluteCaptureTime); |
| 336 | if (new_ids.abs_capture_time) { |
| 337 | rtp_rtcp_module_->RegisterRtpHeaderExtension( |
Danil Chapovalov | d0321c5 | 2021-09-14 10:58:51 | [diff] [blame] | 338 | AbsoluteCaptureTimeExtension::Uri(), new_ids.abs_capture_time); |
Minyue Li | 74dadc1 | 2020-03-05 10:33:13 | [diff] [blame] | 339 | } |
| 340 | } |
| 341 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 342 | if (!ReconfigureSendCodec(new_config)) { |
Mirko Bonadei | 675513b | 2017-11-09 10:09:25 | [diff] [blame] | 343 | RTC_LOG(LS_ERROR) << "Failed to set up send codec state."; |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 344 | } |
| 345 | |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 346 | // Set currently known overhead (used in ANA, opus only). |
| 347 | { |
Markus Handell | 6287280 | 2020-07-06 13:15:07 | [diff] [blame] | 348 | MutexLock lock(&overhead_per_packet_lock_); |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 349 | UpdateOverheadForEncoder(); |
| 350 | } |
| 351 | |
Jakob Ivarsson | d14525e | 2020-03-06 08:49:29 | [diff] [blame] | 352 | channel_send_->CallEncoder([this](AudioEncoder* encoder) { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 353 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Jakob Ivarsson | d14525e | 2020-03-06 08:49:29 | [diff] [blame] | 354 | if (!encoder) { |
| 355 | return; |
| 356 | } |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 357 | frame_length_range_ = encoder->GetFrameLengthRange(); |
| 358 | UpdateCachedTargetAudioBitrateConstraints(); |
Jakob Ivarsson | d14525e | 2020-03-06 08:49:29 | [diff] [blame] | 359 | }); |
| 360 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 361 | if (sending_) { |
| 362 | ReconfigureBitrateObserver(new_config); |
Oskar Sundbom | f85e31b | 2017-12-20 15:38:09 | [diff] [blame] | 363 | } |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 364 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 365 | config_ = new_config; |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 366 | if (!first_time) { |
| 367 | UpdateCachedTargetAudioBitrateConstraints(); |
| 368 | } |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 369 | } |
| 370 | |
solenberg | 3a94154 | 2015-11-16 15:34:50 | [diff] [blame] | 371 | void AudioSendStream::Start() { |
Sebastian Jansson | 8672cac | 2019-03-01 14:57:55 | [diff] [blame] | 372 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 373 | if (sending_) { |
| 374 | return; |
| 375 | } |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 376 | if (!config_.has_dscp && config_.min_bitrate_bps != -1 && |
| 377 | config_.max_bitrate_bps != -1 && |
Sebastian Jansson | cd0eedb | 2019-10-10 11:52:26 | [diff] [blame] | 378 | (allocate_audio_without_feedback_ || TransportSeqNumId(config_) != 0)) { |
Erik Språng | aa59eca | 2019-07-24 12:52:55 | [diff] [blame] | 379 | rtp_transport_->AccountForAudioPacketsInPacedSender(true); |
Sebastian Jansson | c3eb9fd | 2020-01-29 16:42:52 | [diff] [blame] | 380 | if (send_side_bwe_with_overhead_) |
| 381 | rtp_transport_->IncludeOverheadInPacedSender(); |
Sebastian Jansson | b686396 | 2018-10-10 08:23:13 | [diff] [blame] | 382 | rtp_rtcp_module_->SetAsPartOfAllocation(true); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 383 | ConfigureBitrateObserver(); |
Sebastian Jansson | b686396 | 2018-10-10 08:23:13 | [diff] [blame] | 384 | } else { |
| 385 | rtp_rtcp_module_->SetAsPartOfAllocation(false); |
mflodman | 86cc6ff | 2016-07-26 11:44:06 | [diff] [blame] | 386 | } |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 387 | channel_send_->StartSend(); |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 388 | sending_ = true; |
| 389 | audio_state()->AddSendingStream(this, encoder_sample_rate_hz_, |
| 390 | encoder_num_channels_); |
solenberg | 3a94154 | 2015-11-16 15:34:50 | [diff] [blame] | 391 | } |
| 392 | |
| 393 | void AudioSendStream::Stop() { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 394 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 395 | if (!sending_) { |
| 396 | return; |
| 397 | } |
| 398 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 399 | RemoveBitrateObserver(); |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 400 | channel_send_->StopSend(); |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 401 | sending_ = false; |
| 402 | audio_state()->RemoveSendingStream(this); |
| 403 | } |
| 404 | |
| 405 | void AudioSendStream::SendAudioData(std::unique_ptr<AudioFrame> audio_frame) { |
| 406 | RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_); |
Henrik Boström | d2c336f | 2019-07-03 15:11:10 | [diff] [blame] | 407 | RTC_DCHECK_GT(audio_frame->sample_rate_hz_, 0); |
| 408 | double duration = static_cast<double>(audio_frame->samples_per_channel_) / |
| 409 | audio_frame->sample_rate_hz_; |
| 410 | { |
| 411 | // Note: SendAudioData() passes the frame further down the pipeline and it |
| 412 | // may eventually get sent. But this method is invoked even if we are not |
| 413 | // connected, as long as we have an AudioSendStream (created as a result of |
| 414 | // an O/A exchange). This means that we are calculating audio levels whether |
| 415 | // or not we are sending samples. |
| 416 | // TODO(https://crbug.com/webrtc/10771): All "media-source" related stats |
| 417 | // should move from send-streams to the local audio sources or tracks; a |
| 418 | // send-stream should not be required to read the microphone audio levels. |
Markus Handell | 6287280 | 2020-07-06 13:15:07 | [diff] [blame] | 419 | MutexLock lock(&audio_level_lock_); |
Henrik Boström | d2c336f | 2019-07-03 15:11:10 | [diff] [blame] | 420 | audio_level_.ComputeLevel(*audio_frame, duration); |
| 421 | } |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 422 | channel_send_->ProcessAndEncodeAudio(std::move(audio_frame)); |
solenberg | 3a94154 | 2015-11-16 15:34:50 | [diff] [blame] | 423 | } |
| 424 | |
solenberg | ffbbcac | 2016-11-17 13:25:37 | [diff] [blame] | 425 | bool AudioSendStream::SendTelephoneEvent(int payload_type, |
Yves Gerey | 665174f | 2018-06-19 13:03:05 | [diff] [blame] | 426 | int payload_frequency, |
| 427 | int event, |
solenberg | 8842c3e | 2016-03-11 11:06:41 | [diff] [blame] | 428 | int duration_ms) { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 429 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Niels Möller | 8fb1a6a | 2019-03-05 13:29:42 | [diff] [blame] | 430 | channel_send_->SetSendTelephoneEventPayloadType(payload_type, |
| 431 | payload_frequency); |
| 432 | return channel_send_->SendTelephoneEventOutband(event, duration_ms); |
Fredrik Solenberg | b572768 | 2015-12-04 14:22:19 | [diff] [blame] | 433 | } |
| 434 | |
solenberg | 9421853 | 2016-06-16 17:53:22 | [diff] [blame] | 435 | void AudioSendStream::SetMuted(bool muted) { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 436 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 437 | channel_send_->SetInputMute(muted); |
solenberg | 9421853 | 2016-06-16 17:53:22 | [diff] [blame] | 438 | } |
| 439 | |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 440 | webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const { |
Ivo Creusen | 56d46090 | 2017-11-24 16:29:59 | [diff] [blame] | 441 | return GetStats(true); |
| 442 | } |
| 443 | |
| 444 | webrtc::AudioSendStream::Stats AudioSendStream::GetStats( |
| 445 | bool has_remote_tracks) const { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 446 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 447 | webrtc::AudioSendStream::Stats stats; |
| 448 | stats.local_ssrc = config_.rtp.ssrc; |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 449 | stats.target_bitrate_bps = channel_send_->GetBitrate(); |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 450 | |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 451 | webrtc::CallSendStatistics call_stats = channel_send_->GetRTCPStatistics(); |
Niels Möller | ac0a4cb | 2019-10-09 13:01:33 | [diff] [blame] | 452 | stats.payload_bytes_sent = call_stats.payload_bytes_sent; |
| 453 | stats.header_and_padding_bytes_sent = |
| 454 | call_stats.header_and_padding_bytes_sent; |
Henrik Boström | cf96e0f | 2019-04-17 11:51:53 | [diff] [blame] | 455 | stats.retransmitted_bytes_sent = call_stats.retransmitted_bytes_sent; |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 456 | stats.packets_sent = call_stats.packetsSent; |
Henrik Boström | cf96e0f | 2019-04-17 11:51:53 | [diff] [blame] | 457 | stats.retransmitted_packets_sent = call_stats.retransmitted_packets_sent; |
solenberg | 8b85de2 | 2015-11-16 17:48:04 | [diff] [blame] | 458 | // RTT isn't known until a RTCP report is received. Until then, VoiceEngine |
| 459 | // returns 0 to indicate an error value. |
| 460 | if (call_stats.rttMs > 0) { |
| 461 | stats.rtt_ms = call_stats.rttMs; |
| 462 | } |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 463 | if (config_.send_codec_spec) { |
| 464 | const auto& spec = *config_.send_codec_spec; |
| 465 | stats.codec_name = spec.format.name; |
Oskar Sundbom | 2707fb2 | 2017-11-16 09:57:35 | [diff] [blame] | 466 | stats.codec_payload_type = spec.payload_type; |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 467 | |
| 468 | // Get data from the last remote RTCP report. |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 469 | for (const auto& block : channel_send_->GetRemoteRTCPReportBlocks()) { |
solenberg | 8b85de2 | 2015-11-16 17:48:04 | [diff] [blame] | 470 | // Lookup report for send ssrc only. |
| 471 | if (block.source_SSRC == stats.local_ssrc) { |
| 472 | stats.packets_lost = block.cumulative_num_packets_lost; |
| 473 | stats.fraction_lost = Q8ToFloat(block.fraction_lost); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 474 | // Convert timestamps to milliseconds. |
| 475 | if (spec.format.clockrate_hz / 1000 > 0) { |
solenberg | 8b85de2 | 2015-11-16 17:48:04 | [diff] [blame] | 476 | stats.jitter_ms = |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 477 | block.interarrival_jitter / (spec.format.clockrate_hz / 1000); |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 478 | } |
solenberg | 8b85de2 | 2015-11-16 17:48:04 | [diff] [blame] | 479 | break; |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 480 | } |
| 481 | } |
| 482 | } |
| 483 | |
Henrik Boström | d2c336f | 2019-07-03 15:11:10 | [diff] [blame] | 484 | { |
Markus Handell | 6287280 | 2020-07-06 13:15:07 | [diff] [blame] | 485 | MutexLock lock(&audio_level_lock_); |
Henrik Boström | d2c336f | 2019-07-03 15:11:10 | [diff] [blame] | 486 | stats.audio_level = audio_level_.LevelFullRange(); |
| 487 | stats.total_input_energy = audio_level_.TotalEnergy(); |
| 488 | stats.total_input_duration = audio_level_.TotalDuration(); |
| 489 | } |
solenberg | 796b8f9 | 2017-03-02 01:02:23 | [diff] [blame] | 490 | |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 491 | stats.typing_noise_detected = audio_state()->typing_noise_detected(); |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 492 | stats.ana_statistics = channel_send_->GetANAStatistics(); |
Per Åhgren | cc73ed3 | 2020-04-26 21:56:17 | [diff] [blame] | 493 | |
| 494 | AudioProcessing* ap = audio_state_->audio_processing(); |
| 495 | if (ap) { |
| 496 | stats.apm_statistics = ap->GetStatistics(has_remote_tracks); |
| 497 | } |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 498 | |
Henrik Boström | 6e436d1 | 2019-05-27 10:19:33 | [diff] [blame] | 499 | stats.report_block_datas = std::move(call_stats.report_block_datas); |
| 500 | |
Jakob Ivarsson | e91c992 | 2021-07-06 07:55:43 | [diff] [blame] | 501 | stats.nacks_rcvd = call_stats.nacks_rcvd; |
| 502 | |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 503 | return stats; |
| 504 | } |
| 505 | |
Niels Möller | 8fb1a6a | 2019-03-05 13:29:42 | [diff] [blame] | 506 | void AudioSendStream::DeliverRtcp(const uint8_t* packet, size_t length) { |
Erik Språng | 2b4d2f3 | 2020-06-29 14:37:44 | [diff] [blame] | 507 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Niels Möller | 8fb1a6a | 2019-03-05 13:29:42 | [diff] [blame] | 508 | channel_send_->ReceivedRTCPPacket(packet, length); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 509 | |
| 510 | { |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 511 | // Poll if overhead has changed, which it can do if ack triggers us to stop |
| 512 | // sending mid/rid. |
Markus Handell | 6287280 | 2020-07-06 13:15:07 | [diff] [blame] | 513 | MutexLock lock(&overhead_per_packet_lock_); |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 514 | UpdateOverheadForEncoder(); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 515 | } |
| 516 | UpdateCachedTargetAudioBitrateConstraints(); |
pbos | 1ba8d39 | 2016-05-02 03:18:34 | [diff] [blame] | 517 | } |
| 518 | |
Sebastian Jansson | c0e4d45 | 2018-10-25 13:08:32 | [diff] [blame] | 519 | uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) { |
Markus Handell | 3907e7b | 2021-06-01 07:07:20 | [diff] [blame] | 520 | RTC_DCHECK_RUN_ON(rtp_transport_queue_); |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 521 | |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 522 | // Pick a target bitrate between the constraints. Overrules the allocator if |
| 523 | // it 1) allocated a bitrate of zero to disable the stream or 2) allocated a |
| 524 | // higher than max to allow for e.g. extra FEC. |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 525 | RTC_DCHECK(cached_constraints_.has_value()); |
| 526 | update.target_bitrate.Clamp(cached_constraints_->min, |
| 527 | cached_constraints_->max); |
| 528 | update.stable_target_bitrate.Clamp(cached_constraints_->min, |
| 529 | cached_constraints_->max); |
mflodman | 86cc6ff | 2016-07-26 11:44:06 | [diff] [blame] | 530 | |
Sebastian Jansson | 254d869 | 2018-11-21 18:19:00 | [diff] [blame] | 531 | channel_send_->OnBitrateAllocation(update); |
mflodman | 86cc6ff | 2016-07-26 11:44:06 | [diff] [blame] | 532 | |
| 533 | // The amount of audio protection is not exposed by the encoder, hence |
| 534 | // always returning 0. |
| 535 | return 0; |
| 536 | } |
| 537 | |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 538 | void AudioSendStream::SetTransportOverhead( |
| 539 | int transport_overhead_per_packet_bytes) { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 540 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
| 541 | { |
| 542 | MutexLock lock(&overhead_per_packet_lock_); |
| 543 | transport_overhead_per_packet_bytes_ = transport_overhead_per_packet_bytes; |
| 544 | UpdateOverheadForEncoder(); |
| 545 | } |
| 546 | UpdateCachedTargetAudioBitrateConstraints(); |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 547 | } |
| 548 | |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 549 | void AudioSendStream::UpdateOverheadForEncoder() { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 550 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Erik Språng | cf6544a | 2020-05-13 12:43:11 | [diff] [blame] | 551 | size_t overhead_per_packet_bytes = GetPerPacketOverheadBytes(); |
| 552 | if (overhead_per_packet_ == overhead_per_packet_bytes) { |
| 553 | return; |
Bjorn A Mellem | 413ccc4 | 2019-04-26 22:41:05 | [diff] [blame] | 554 | } |
Erik Språng | cf6544a | 2020-05-13 12:43:11 | [diff] [blame] | 555 | overhead_per_packet_ = overhead_per_packet_bytes; |
| 556 | |
Sebastian Jansson | 14a7cf9 | 2019-02-13 14:11:42 | [diff] [blame] | 557 | channel_send_->CallEncoder([&](AudioEncoder* encoder) { |
| 558 | encoder->OnReceivedOverhead(overhead_per_packet_bytes); |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 559 | }); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 560 | if (total_packet_overhead_bytes_ != overhead_per_packet_bytes) { |
| 561 | total_packet_overhead_bytes_ = overhead_per_packet_bytes; |
| 562 | if (registered_with_allocator_) { |
| 563 | ConfigureBitrateObserver(); |
Sebastian Jansson | 8672cac | 2019-03-01 14:57:55 | [diff] [blame] | 564 | } |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 565 | } |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 566 | } |
| 567 | |
| 568 | size_t AudioSendStream::TestOnlyGetPerPacketOverheadBytes() const { |
Markus Handell | 6287280 | 2020-07-06 13:15:07 | [diff] [blame] | 569 | MutexLock lock(&overhead_per_packet_lock_); |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 570 | return GetPerPacketOverheadBytes(); |
| 571 | } |
| 572 | |
| 573 | size_t AudioSendStream::GetPerPacketOverheadBytes() const { |
| 574 | return transport_overhead_per_packet_bytes_ + |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 575 | rtp_rtcp_module_->ExpectedPerPacketOverhead(); |
michaelt | 79e0588 | 2016-11-08 10:50:09 | [diff] [blame] | 576 | } |
| 577 | |
ossu | c3d4b48 | 2017-05-23 13:07:11 | [diff] [blame] | 578 | RtpState AudioSendStream::GetRtpState() const { |
| 579 | return rtp_rtcp_module_->GetRtpState(); |
| 580 | } |
| 581 | |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 582 | const voe::ChannelSendInterface* AudioSendStream::GetChannel() const { |
| 583 | return channel_send_.get(); |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 584 | } |
| 585 | |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 586 | internal::AudioState* AudioSendStream::audio_state() { |
| 587 | internal::AudioState* audio_state = |
| 588 | static_cast<internal::AudioState*>(audio_state_.get()); |
| 589 | RTC_DCHECK(audio_state); |
| 590 | return audio_state; |
| 591 | } |
| 592 | |
| 593 | const internal::AudioState* AudioSendStream::audio_state() const { |
| 594 | internal::AudioState* audio_state = |
| 595 | static_cast<internal::AudioState*>(audio_state_.get()); |
| 596 | RTC_DCHECK(audio_state); |
| 597 | return audio_state; |
| 598 | } |
| 599 | |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 600 | void AudioSendStream::StoreEncoderProperties(int sample_rate_hz, |
| 601 | size_t num_channels) { |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 602 | encoder_sample_rate_hz_ = sample_rate_hz; |
| 603 | encoder_num_channels_ = num_channels; |
| 604 | if (sending_) { |
| 605 | // Update AudioState's information about the stream. |
| 606 | audio_state()->AddSendingStream(this, sample_rate_hz, num_channels); |
| 607 | } |
| 608 | } |
| 609 | |
minyue | 7a97344 | 2016-10-20 10:27:12 | [diff] [blame] | 610 | // Apply current codec settings to a single voe::Channel used for sending. |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 611 | bool AudioSendStream::SetupSendCodec(const Config& new_config) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 612 | RTC_DCHECK(new_config.send_codec_spec); |
| 613 | const auto& spec = *new_config.send_codec_spec; |
minyue | 48368ad | 2017-05-10 11:06:11 | [diff] [blame] | 614 | |
| 615 | RTC_DCHECK(new_config.encoder_factory); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 616 | std::unique_ptr<AudioEncoder> encoder = |
Karl Wiberg | 77490b9 | 2018-03-21 14:18:42 | [diff] [blame] | 617 | new_config.encoder_factory->MakeAudioEncoder( |
| 618 | spec.payload_type, spec.format, new_config.codec_pair_id); |
minyue | 7a97344 | 2016-10-20 10:27:12 | [diff] [blame] | 619 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 620 | if (!encoder) { |
Jonas Olsson | abbe841 | 2018-04-03 11:40:05 | [diff] [blame] | 621 | RTC_DLOG(LS_ERROR) << "Unable to create encoder for " |
| 622 | << rtc::ToString(spec.format); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 623 | return false; |
| 624 | } |
Alex Narest | bbbe4e1 | 2018-07-13 08:32:58 | [diff] [blame] | 625 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 626 | // If a bitrate has been specified for the codec, use it over the |
| 627 | // codec's default. |
Christoffer Rodbro | 110c64b | 2019-03-06 08:51:08 | [diff] [blame] | 628 | if (spec.target_bitrate_bps) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 629 | encoder->OnReceivedTargetAudioBitrate(*spec.target_bitrate_bps); |
minyue | 7a97344 | 2016-10-20 10:27:12 | [diff] [blame] | 630 | } |
| 631 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 632 | // Enable ANA if configured (currently only used by Opus). |
Mirko Bonadei | 4356490 | 2020-01-29 15:29:36 | [diff] [blame] | 633 | if (new_config.audio_network_adaptor_config) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 634 | if (encoder->EnableAudioNetworkAdaptor( |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 635 | *new_config.audio_network_adaptor_config, event_log_)) { |
Jakob Ivarsson | ed97116 | 2020-08-11 12:05:07 | [diff] [blame] | 636 | RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC " |
| 637 | << new_config.rtp.ssrc; |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 638 | } else { |
Jakob Ivarsson | ed97116 | 2020-08-11 12:05:07 | [diff] [blame] | 639 | RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC " |
| 640 | << new_config.rtp.ssrc; |
minyue | 6b825df | 2016-10-31 11:08:32 | [diff] [blame] | 641 | } |
minyue | 7a97344 | 2016-10-20 10:27:12 | [diff] [blame] | 642 | } |
| 643 | |
Philipp Hancke | 1a49756 | 2020-05-26 17:12:31 | [diff] [blame] | 644 | // Wrap the encoder in an AudioEncoderCNG, if VAD is enabled. |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 645 | if (spec.cng_payload_type) { |
Karl Wiberg | 2365936 | 2018-11-01 10:13:44 | [diff] [blame] | 646 | AudioEncoderCngConfig cng_config; |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 647 | cng_config.num_channels = encoder->NumChannels(); |
| 648 | cng_config.payload_type = *spec.cng_payload_type; |
| 649 | cng_config.speech_encoder = std::move(encoder); |
| 650 | cng_config.vad_mode = Vad::kVadNormal; |
Karl Wiberg | 2365936 | 2018-11-01 10:13:44 | [diff] [blame] | 651 | encoder = CreateComfortNoiseEncoder(std::move(cng_config)); |
ossu | 3b9ff38 | 2017-04-27 15:03:42 | [diff] [blame] | 652 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 653 | RegisterCngPayloadType(*spec.cng_payload_type, |
| 654 | new_config.send_codec_spec->format.clockrate_hz); |
minyue | 7a97344 | 2016-10-20 10:27:12 | [diff] [blame] | 655 | } |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 656 | |
Philipp Hancke | edcd966 | 2020-06-24 10:52:42 | [diff] [blame] | 657 | // Wrap the encoder in a RED encoder, if RED is enabled. |
| 658 | if (spec.red_payload_type) { |
| 659 | AudioEncoderCopyRed::Config red_config; |
| 660 | red_config.payload_type = *spec.red_payload_type; |
| 661 | red_config.speech_encoder = std::move(encoder); |
| 662 | encoder = std::make_unique<AudioEncoderCopyRed>(std::move(red_config)); |
| 663 | } |
| 664 | |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 665 | // Set currently known overhead (used in ANA, opus only). |
| 666 | // If overhead changes later, it will be updated in UpdateOverheadForEncoder. |
| 667 | { |
Markus Handell | 6287280 | 2020-07-06 13:15:07 | [diff] [blame] | 668 | MutexLock lock(&overhead_per_packet_lock_); |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 669 | size_t overhead = GetPerPacketOverheadBytes(); |
| 670 | if (overhead > 0) { |
| 671 | encoder->OnReceivedOverhead(overhead); |
Bjorn A Mellem | 413ccc4 | 2019-04-26 22:41:05 | [diff] [blame] | 672 | } |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 673 | } |
| 674 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 675 | StoreEncoderProperties(encoder->SampleRateHz(), encoder->NumChannels()); |
| 676 | channel_send_->SetEncoder(new_config.send_codec_spec->payload_type, |
| 677 | std::move(encoder)); |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 678 | |
minyue | 7a97344 | 2016-10-20 10:27:12 | [diff] [blame] | 679 | return true; |
| 680 | } |
| 681 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 682 | bool AudioSendStream::ReconfigureSendCodec(const Config& new_config) { |
| 683 | const auto& old_config = config_; |
minyue-webrtc | 8de1826 | 2017-07-26 12:18:40 | [diff] [blame] | 684 | |
| 685 | if (!new_config.send_codec_spec) { |
| 686 | // We cannot de-configure a send codec. So we will do nothing. |
| 687 | // By design, the send codec should have not been configured. |
| 688 | RTC_DCHECK(!old_config.send_codec_spec); |
| 689 | return true; |
| 690 | } |
| 691 | |
| 692 | if (new_config.send_codec_spec == old_config.send_codec_spec && |
| 693 | new_config.audio_network_adaptor_config == |
| 694 | old_config.audio_network_adaptor_config) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 695 | return true; |
| 696 | } |
| 697 | |
| 698 | // If we have no encoder, or the format or payload type's changed, create a |
| 699 | // new encoder. |
| 700 | if (!old_config.send_codec_spec || |
| 701 | new_config.send_codec_spec->format != |
| 702 | old_config.send_codec_spec->format || |
| 703 | new_config.send_codec_spec->payload_type != |
Philipp Hancke | 6144b84 | 2021-06-04 11:49:27 | [diff] [blame] | 704 | old_config.send_codec_spec->payload_type || |
| 705 | new_config.send_codec_spec->red_payload_type != |
| 706 | old_config.send_codec_spec->red_payload_type) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 707 | return SetupSendCodec(new_config); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 708 | } |
| 709 | |
Danil Chapovalov | b9b146c | 2018-06-15 10:28:07 | [diff] [blame] | 710 | const absl::optional<int>& new_target_bitrate_bps = |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 711 | new_config.send_codec_spec->target_bitrate_bps; |
| 712 | // If a bitrate has been specified for the codec, use it over the |
| 713 | // codec's default. |
Christoffer Rodbro | 110c64b | 2019-03-06 08:51:08 | [diff] [blame] | 714 | if (new_target_bitrate_bps && |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 715 | new_target_bitrate_bps != |
| 716 | old_config.send_codec_spec->target_bitrate_bps) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 717 | channel_send_->CallEncoder([&](AudioEncoder* encoder) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 718 | encoder->OnReceivedTargetAudioBitrate(*new_target_bitrate_bps); |
| 719 | }); |
| 720 | } |
| 721 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 722 | ReconfigureANA(new_config); |
| 723 | ReconfigureCNG(new_config); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 724 | |
| 725 | return true; |
| 726 | } |
| 727 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 728 | void AudioSendStream::ReconfigureANA(const Config& new_config) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 729 | if (new_config.audio_network_adaptor_config == |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 730 | config_.audio_network_adaptor_config) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 731 | return; |
| 732 | } |
Mirko Bonadei | 4356490 | 2020-01-29 15:29:36 | [diff] [blame] | 733 | if (new_config.audio_network_adaptor_config) { |
Jakob Ivarsson | fde2b24 | 2020-08-20 14:48:49 | [diff] [blame] | 734 | // This lock needs to be acquired before CallEncoder, since it aquires |
| 735 | // another lock and we need to maintain the same order at all call sites to |
| 736 | // avoid deadlock. |
| 737 | MutexLock lock(&overhead_per_packet_lock_); |
| 738 | size_t overhead = GetPerPacketOverheadBytes(); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 739 | channel_send_->CallEncoder([&](AudioEncoder* encoder) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 740 | if (encoder->EnableAudioNetworkAdaptor( |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 741 | *new_config.audio_network_adaptor_config, event_log_)) { |
Jakob Ivarsson | ed97116 | 2020-08-11 12:05:07 | [diff] [blame] | 742 | RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC " |
| 743 | << new_config.rtp.ssrc; |
Jakob Ivarsson | fde2b24 | 2020-08-20 14:48:49 | [diff] [blame] | 744 | if (overhead > 0) { |
| 745 | encoder->OnReceivedOverhead(overhead); |
| 746 | } |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 747 | } else { |
Jakob Ivarsson | ed97116 | 2020-08-11 12:05:07 | [diff] [blame] | 748 | RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC " |
| 749 | << new_config.rtp.ssrc; |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 750 | } |
| 751 | }); |
| 752 | } else { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 753 | channel_send_->CallEncoder( |
Sebastian Jansson | 14a7cf9 | 2019-02-13 14:11:42 | [diff] [blame] | 754 | [&](AudioEncoder* encoder) { encoder->DisableAudioNetworkAdaptor(); }); |
Jakob Ivarsson | ed97116 | 2020-08-11 12:05:07 | [diff] [blame] | 755 | RTC_LOG(LS_INFO) << "Audio network adaptor disabled on SSRC " |
| 756 | << new_config.rtp.ssrc; |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 757 | } |
| 758 | } |
| 759 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 760 | void AudioSendStream::ReconfigureCNG(const Config& new_config) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 761 | if (new_config.send_codec_spec->cng_payload_type == |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 762 | config_.send_codec_spec->cng_payload_type) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 763 | return; |
| 764 | } |
| 765 | |
ossu | 3b9ff38 | 2017-04-27 15:03:42 | [diff] [blame] | 766 | // Register the CNG payload type if it's been added, don't do anything if CNG |
| 767 | // is removed. Payload types must not be redefined. |
| 768 | if (new_config.send_codec_spec->cng_payload_type) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 769 | RegisterCngPayloadType(*new_config.send_codec_spec->cng_payload_type, |
| 770 | new_config.send_codec_spec->format.clockrate_hz); |
ossu | 3b9ff38 | 2017-04-27 15:03:42 | [diff] [blame] | 771 | } |
| 772 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 773 | // Wrap or unwrap the encoder in an AudioEncoderCNG. |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 774 | channel_send_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder_ptr) { |
| 775 | std::unique_ptr<AudioEncoder> old_encoder(std::move(*encoder_ptr)); |
| 776 | auto sub_encoders = old_encoder->ReclaimContainedEncoders(); |
| 777 | if (!sub_encoders.empty()) { |
| 778 | // Replace enc with its sub encoder. We need to put the sub |
| 779 | // encoder in a temporary first, since otherwise the old value |
| 780 | // of enc would be destroyed before the new value got assigned, |
| 781 | // which would be bad since the new value is a part of the old |
| 782 | // value. |
| 783 | auto tmp = std::move(sub_encoders[0]); |
| 784 | old_encoder = std::move(tmp); |
| 785 | } |
| 786 | if (new_config.send_codec_spec->cng_payload_type) { |
| 787 | AudioEncoderCngConfig config; |
| 788 | config.speech_encoder = std::move(old_encoder); |
| 789 | config.num_channels = config.speech_encoder->NumChannels(); |
| 790 | config.payload_type = *new_config.send_codec_spec->cng_payload_type; |
| 791 | config.vad_mode = Vad::kVadNormal; |
| 792 | *encoder_ptr = CreateComfortNoiseEncoder(std::move(config)); |
| 793 | } else { |
| 794 | *encoder_ptr = std::move(old_encoder); |
| 795 | } |
| 796 | }); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 797 | } |
| 798 | |
| 799 | void AudioSendStream::ReconfigureBitrateObserver( |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 800 | const webrtc::AudioSendStream::Config& new_config) { |
| 801 | // Since the Config's default is for both of these to be -1, this test will |
| 802 | // allow us to configure the bitrate observer if the new config has bitrate |
| 803 | // limits set, but would only have us call RemoveBitrateObserver if we were |
| 804 | // previously configured with bitrate limits. |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 805 | if (config_.min_bitrate_bps == new_config.min_bitrate_bps && |
| 806 | config_.max_bitrate_bps == new_config.max_bitrate_bps && |
| 807 | config_.bitrate_priority == new_config.bitrate_priority && |
Jakob Ivarsson | 47a03e8 | 2020-11-23 14:05:44 | [diff] [blame] | 808 | TransportSeqNumId(config_) == TransportSeqNumId(new_config) && |
Jakob Ivarsson | d14525e | 2020-03-06 08:49:29 | [diff] [blame] | 809 | config_.audio_network_adaptor_config == |
| 810 | new_config.audio_network_adaptor_config) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 811 | return; |
| 812 | } |
| 813 | |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 814 | if (!new_config.has_dscp && new_config.min_bitrate_bps != -1 && |
Sebastian Jansson | cd0eedb | 2019-10-10 11:52:26 | [diff] [blame] | 815 | new_config.max_bitrate_bps != -1 && TransportSeqNumId(new_config) != 0) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 816 | rtp_transport_->AccountForAudioPacketsInPacedSender(true); |
Sebastian Jansson | c3eb9fd | 2020-01-29 16:42:52 | [diff] [blame] | 817 | if (send_side_bwe_with_overhead_) |
| 818 | rtp_transport_->IncludeOverheadInPacedSender(); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 819 | // We may get a callback immediately as the observer is registered, so |
| 820 | // make sure the bitrate limits in config_ are up-to-date. |
| 821 | config_.min_bitrate_bps = new_config.min_bitrate_bps; |
| 822 | config_.max_bitrate_bps = new_config.max_bitrate_bps; |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 823 | |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 824 | config_.bitrate_priority = new_config.bitrate_priority; |
| 825 | ConfigureBitrateObserver(); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 826 | rtp_rtcp_module_->SetAsPartOfAllocation(true); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 827 | } else { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 828 | rtp_transport_->AccountForAudioPacketsInPacedSender(false); |
| 829 | RemoveBitrateObserver(); |
| 830 | rtp_rtcp_module_->SetAsPartOfAllocation(false); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 831 | } |
| 832 | } |
| 833 | |
Sebastian Jansson | 8672cac | 2019-03-01 14:57:55 | [diff] [blame] | 834 | void AudioSendStream::ConfigureBitrateObserver() { |
| 835 | // This either updates the current observer or adds a new observer. |
| 836 | // TODO(srte): Add overhead compensation here. |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 837 | auto constraints = GetMinMaxBitrateConstraints(); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 838 | RTC_DCHECK(constraints.has_value()); |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 839 | |
Sebastian Jansson | 0429f78 | 2019-10-03 16:32:45 | [diff] [blame] | 840 | DataRate priority_bitrate = allocation_settings_.priority_bitrate; |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 841 | if (send_side_bwe_with_overhead_) { |
Sebastian Jansson | 0429f78 | 2019-10-03 16:32:45 | [diff] [blame] | 842 | if (use_legacy_overhead_calculation_) { |
| 843 | // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12) |
| 844 | constexpr int kOverheadPerPacket = 20 + 8 + 10 + 12; |
Danil Chapovalov | 0c626af | 2020-02-10 10:16:00 | [diff] [blame] | 845 | const TimeDelta kMinPacketDuration = TimeDelta::Millis(20); |
Sebastian Jansson | 0429f78 | 2019-10-03 16:32:45 | [diff] [blame] | 846 | DataRate max_overhead = |
Danil Chapovalov | cad3e0e | 2020-02-17 17:46:07 | [diff] [blame] | 847 | DataSize::Bytes(kOverheadPerPacket) / kMinPacketDuration; |
Sebastian Jansson | 0429f78 | 2019-10-03 16:32:45 | [diff] [blame] | 848 | priority_bitrate += max_overhead; |
| 849 | } else { |
| 850 | RTC_DCHECK(frame_length_range_); |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 851 | const DataSize overhead_per_packet = |
Danil Chapovalov | cad3e0e | 2020-02-17 17:46:07 | [diff] [blame] | 852 | DataSize::Bytes(total_packet_overhead_bytes_); |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 853 | DataRate min_overhead = overhead_per_packet / frame_length_range_->second; |
Jakob Ivarsson | 01ab084 | 2020-03-06 08:59:56 | [diff] [blame] | 854 | priority_bitrate += min_overhead; |
Sebastian Jansson | 0429f78 | 2019-10-03 16:32:45 | [diff] [blame] | 855 | } |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 856 | } |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 857 | if (allocation_settings_.priority_bitrate_raw) |
| 858 | priority_bitrate = *allocation_settings_.priority_bitrate_raw; |
| 859 | |
Markus Handell | 3907e7b | 2021-06-01 07:07:20 | [diff] [blame] | 860 | rtp_transport_queue_->PostTask([this, constraints, priority_bitrate, |
| 861 | config_bitrate_priority = |
| 862 | config_.bitrate_priority] { |
| 863 | RTC_DCHECK_RUN_ON(rtp_transport_queue_); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 864 | bitrate_allocator_->AddObserver( |
| 865 | this, |
| 866 | MediaStreamAllocationConfig{ |
| 867 | constraints->min.bps<uint32_t>(), constraints->max.bps<uint32_t>(), |
| 868 | 0, priority_bitrate.bps(), true, |
| 869 | allocation_settings_.bitrate_priority.value_or( |
| 870 | config_bitrate_priority)}); |
| 871 | }); |
Jakob Ivarsson | d14525e | 2020-03-06 08:49:29 | [diff] [blame] | 872 | registered_with_allocator_ = true; |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 873 | } |
| 874 | |
| 875 | void AudioSendStream::RemoveBitrateObserver() { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 876 | registered_with_allocator_ = false; |
Niels Möller | c572ff3 | 2018-11-07 07:43:50 | [diff] [blame] | 877 | rtc::Event thread_sync_event; |
Markus Handell | 3907e7b | 2021-06-01 07:07:20 | [diff] [blame] | 878 | rtp_transport_queue_->PostTask([this, &thread_sync_event] { |
| 879 | RTC_DCHECK_RUN_ON(rtp_transport_queue_); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 880 | bitrate_allocator_->RemoveObserver(this); |
| 881 | thread_sync_event.Set(); |
| 882 | }); |
| 883 | thread_sync_event.Wait(rtc::Event::kForever); |
| 884 | } |
| 885 | |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 886 | absl::optional<AudioSendStream::TargetAudioBitrateConstraints> |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 887 | AudioSendStream::GetMinMaxBitrateConstraints() const { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 888 | if (config_.min_bitrate_bps < 0 || config_.max_bitrate_bps < 0) { |
| 889 | RTC_LOG(LS_WARNING) << "Config is invalid: min_bitrate_bps=" |
| 890 | << config_.min_bitrate_bps |
| 891 | << "; max_bitrate_bps=" << config_.max_bitrate_bps |
| 892 | << "; both expected greater or equal to 0"; |
| 893 | return absl::nullopt; |
| 894 | } |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 895 | TargetAudioBitrateConstraints constraints{ |
Danil Chapovalov | cad3e0e | 2020-02-17 17:46:07 | [diff] [blame] | 896 | DataRate::BitsPerSec(config_.min_bitrate_bps), |
| 897 | DataRate::BitsPerSec(config_.max_bitrate_bps)}; |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 898 | |
| 899 | // If bitrates were explicitly overriden via field trial, use those values. |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 900 | if (allocation_settings_.min_bitrate) |
| 901 | constraints.min = *allocation_settings_.min_bitrate; |
| 902 | if (allocation_settings_.max_bitrate) |
| 903 | constraints.max = *allocation_settings_.max_bitrate; |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 904 | |
Sebastian Jansson | 62aee93 | 2019-10-02 10:27:06 | [diff] [blame] | 905 | RTC_DCHECK_GE(constraints.min, DataRate::Zero()); |
| 906 | RTC_DCHECK_GE(constraints.max, DataRate::Zero()); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 907 | if (constraints.max < constraints.min) { |
| 908 | RTC_LOG(LS_WARNING) << "TargetAudioBitrateConstraints::max is less than " |
| 909 | << "TargetAudioBitrateConstraints::min"; |
| 910 | return absl::nullopt; |
| 911 | } |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 912 | if (send_side_bwe_with_overhead_) { |
Sebastian Jansson | 62aee93 | 2019-10-02 10:27:06 | [diff] [blame] | 913 | if (use_legacy_overhead_calculation_) { |
| 914 | // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12) |
Danil Chapovalov | cad3e0e | 2020-02-17 17:46:07 | [diff] [blame] | 915 | const DataSize kOverheadPerPacket = DataSize::Bytes(20 + 8 + 10 + 12); |
Sebastian Jansson | 62aee93 | 2019-10-02 10:27:06 | [diff] [blame] | 916 | const TimeDelta kMaxFrameLength = |
Danil Chapovalov | 0c626af | 2020-02-10 10:16:00 | [diff] [blame] | 917 | TimeDelta::Millis(60); // Based on Opus spec |
Sebastian Jansson | 62aee93 | 2019-10-02 10:27:06 | [diff] [blame] | 918 | const DataRate kMinOverhead = kOverheadPerPacket / kMaxFrameLength; |
| 919 | constraints.min += kMinOverhead; |
| 920 | constraints.max += kMinOverhead; |
| 921 | } else { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 922 | if (!frame_length_range_.has_value()) { |
| 923 | RTC_LOG(LS_WARNING) << "frame_length_range_ is not set"; |
| 924 | return absl::nullopt; |
| 925 | } |
Sebastian Jansson | 62aee93 | 2019-10-02 10:27:06 | [diff] [blame] | 926 | const DataSize kOverheadPerPacket = |
Danil Chapovalov | cad3e0e | 2020-02-17 17:46:07 | [diff] [blame] | 927 | DataSize::Bytes(total_packet_overhead_bytes_); |
Sebastian Jansson | 62aee93 | 2019-10-02 10:27:06 | [diff] [blame] | 928 | constraints.min += kOverheadPerPacket / frame_length_range_->second; |
| 929 | constraints.max += kOverheadPerPacket / frame_length_range_->first; |
| 930 | } |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 931 | } |
| 932 | return constraints; |
| 933 | } |
| 934 | |
ossu | 3b9ff38 | 2017-04-27 15:03:42 | [diff] [blame] | 935 | void AudioSendStream::RegisterCngPayloadType(int payload_type, |
| 936 | int clockrate_hz) { |
Niels Möller | ee5ccbc | 2019-03-06 15:47:29 | [diff] [blame] | 937 | channel_send_->RegisterCngPayloadType(payload_type, clockrate_hz); |
ossu | 3b9ff38 | 2017-04-27 15:03:42 | [diff] [blame] | 938 | } |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 939 | |
| 940 | void AudioSendStream::UpdateCachedTargetAudioBitrateConstraints() { |
| 941 | absl::optional<AudioSendStream::TargetAudioBitrateConstraints> |
| 942 | new_constraints = GetMinMaxBitrateConstraints(); |
| 943 | if (!new_constraints.has_value()) { |
| 944 | return; |
| 945 | } |
Markus Handell | 3907e7b | 2021-06-01 07:07:20 | [diff] [blame] | 946 | rtp_transport_queue_->PostTask([this, new_constraints]() { |
| 947 | RTC_DCHECK_RUN_ON(rtp_transport_queue_); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 948 | cached_constraints_ = new_constraints; |
| 949 | }); |
| 950 | } |
| 951 | |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 952 | } // namespace internal |
| 953 | } // namespace webrtc |