solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 11 | #include "audio/audio_send_stream.h" |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 12 | |
Mirko Bonadei | 317a1f0 | 2019-09-17 15:06:18 | [diff] [blame] | 13 | #include <memory> |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 14 | #include <string> |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 15 | #include <utility> |
| 16 | #include <vector> |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 17 | |
Yves Gerey | 988cc08 | 2018-10-23 10:03:01 | [diff] [blame] | 18 | #include "api/audio_codecs/audio_encoder.h" |
| 19 | #include "api/audio_codecs/audio_encoder_factory.h" |
| 20 | #include "api/audio_codecs/audio_format.h" |
| 21 | #include "api/call/transport.h" |
Steve Anton | 10542f2 | 2019-01-11 17:11:00 | [diff] [blame] | 22 | #include "api/crypto/frame_encryptor_interface.h" |
Artem Titov | 741daaf | 2019-03-21 13:37:36 | [diff] [blame] | 23 | #include "api/function_view.h" |
Danil Chapovalov | 83bbe91 | 2019-08-07 10:24:53 | [diff] [blame] | 24 | #include "api/rtc_event_log/rtc_event_log.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 25 | #include "audio/audio_state.h" |
Yves Gerey | 988cc08 | 2018-10-23 10:03:01 | [diff] [blame] | 26 | #include "audio/channel_send.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 27 | #include "audio/conversion.h" |
Yves Gerey | 988cc08 | 2018-10-23 10:03:01 | [diff] [blame] | 28 | #include "call/rtp_config.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 29 | #include "call/rtp_transport_controller_send_interface.h" |
Yves Gerey | 988cc08 | 2018-10-23 10:03:01 | [diff] [blame] | 30 | #include "common_audio/vad/include/vad.h" |
Oskar Sundbom | 56ef305 | 2018-10-30 15:11:02 | [diff] [blame] | 31 | #include "logging/rtc_event_log/events/rtc_event_audio_send_stream_config.h" |
Oskar Sundbom | 56ef305 | 2018-10-30 15:11:02 | [diff] [blame] | 32 | #include "logging/rtc_event_log/rtc_stream_config.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 33 | #include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" |
Philipp Hancke | edcd966 | 2020-06-24 10:52:42 | [diff] [blame] | 34 | #include "modules/audio_coding/codecs/red/audio_encoder_copy_red.h" |
Yves Gerey | 988cc08 | 2018-10-23 10:03:01 | [diff] [blame] | 35 | #include "modules/audio_processing/include/audio_processing.h" |
Sebastian Jansson | 6298b56 | 2020-01-14 16:55:19 | [diff] [blame] | 36 | #include "modules/rtp_rtcp/source/rtp_header_extensions.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 37 | #include "rtc_base/checks.h" |
| 38 | #include "rtc_base/event.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 39 | #include "rtc_base/logging.h" |
Jonas Olsson | abbe841 | 2018-04-03 11:40:05 | [diff] [blame] | 40 | #include "rtc_base/strings/audio_format_to_string.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 41 | #include "rtc_base/task_queue.h" |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 42 | |
| 43 | namespace webrtc { |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 44 | namespace { |
elad.alon | d12a8e1 | 2017-03-23 18:04:48 | [diff] [blame] | 45 | |
Oskar Sundbom | 56ef305 | 2018-10-30 15:11:02 | [diff] [blame] | 46 | void UpdateEventLogStreamConfig(RtcEventLog* event_log, |
| 47 | const AudioSendStream::Config& config, |
| 48 | const AudioSendStream::Config* old_config) { |
| 49 | using SendCodecSpec = AudioSendStream::Config::SendCodecSpec; |
| 50 | // Only update if any of the things we log have changed. |
| 51 | auto payload_types_equal = [](const absl::optional<SendCodecSpec>& a, |
| 52 | const absl::optional<SendCodecSpec>& b) { |
| 53 | if (a.has_value() && b.has_value()) { |
| 54 | return a->format.name == b->format.name && |
| 55 | a->payload_type == b->payload_type; |
| 56 | } |
| 57 | return !a.has_value() && !b.has_value(); |
| 58 | }; |
| 59 | |
| 60 | if (old_config && config.rtp.ssrc == old_config->rtp.ssrc && |
| 61 | config.rtp.extensions == old_config->rtp.extensions && |
| 62 | payload_types_equal(config.send_codec_spec, |
| 63 | old_config->send_codec_spec)) { |
| 64 | return; |
| 65 | } |
| 66 | |
Mirko Bonadei | 317a1f0 | 2019-09-17 15:06:18 | [diff] [blame] | 67 | auto rtclog_config = std::make_unique<rtclog::StreamConfig>(); |
Oskar Sundbom | 56ef305 | 2018-10-30 15:11:02 | [diff] [blame] | 68 | rtclog_config->local_ssrc = config.rtp.ssrc; |
| 69 | rtclog_config->rtp_extensions = config.rtp.extensions; |
| 70 | if (config.send_codec_spec) { |
| 71 | rtclog_config->codecs.emplace_back(config.send_codec_spec->format.name, |
| 72 | config.send_codec_spec->payload_type, 0); |
| 73 | } |
Mirko Bonadei | 317a1f0 | 2019-09-17 15:06:18 | [diff] [blame] | 74 | event_log->Log(std::make_unique<RtcEventAudioSendStreamConfig>( |
Oskar Sundbom | 56ef305 | 2018-10-30 15:11:02 | [diff] [blame] | 75 | std::move(rtclog_config))); |
| 76 | } |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 77 | } // namespace |
| 78 | |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 79 | constexpr char AudioAllocationConfig::kKey[]; |
| 80 | |
| 81 | std::unique_ptr<StructParametersParser> AudioAllocationConfig::Parser() { |
| 82 | return StructParametersParser::Create( // |
| 83 | "min", &min_bitrate, // |
| 84 | "max", &max_bitrate, // |
| 85 | "prio_rate", &priority_bitrate, // |
| 86 | "prio_rate_raw", &priority_bitrate_raw, // |
| 87 | "rate_prio", &bitrate_priority); |
| 88 | } |
| 89 | |
Jonas Oreland | a943e73 | 2022-03-16 12:50:58 | [diff] [blame^] | 90 | AudioAllocationConfig::AudioAllocationConfig( |
| 91 | const WebRtcKeyValueConfig& field_trials) { |
| 92 | Parser()->Parse(field_trials.Lookup(kKey)); |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 93 | if (priority_bitrate_raw && !priority_bitrate.IsZero()) { |
| 94 | RTC_LOG(LS_WARNING) << "'priority_bitrate' and '_raw' are mutually " |
| 95 | "exclusive but both were configured."; |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | namespace internal { |
solenberg | 566ef24 | 2015-11-06 23:34:49 | [diff] [blame] | 100 | AudioSendStream::AudioSendStream( |
Sebastian Jansson | 977b335 | 2019-03-04 16:43:34 | [diff] [blame] | 101 | Clock* clock, |
solenberg | 566ef24 | 2015-11-06 23:34:49 | [diff] [blame] | 102 | const webrtc::AudioSendStream::Config& config, |
Stefan Holmer | b86d4e4 | 2015-12-07 09:26:18 | [diff] [blame] | 103 | const rtc::scoped_refptr<webrtc::AudioState>& audio_state, |
Sebastian Jansson | 44dd9f2 | 2019-03-08 13:50:30 | [diff] [blame] | 104 | TaskQueueFactory* task_queue_factory, |
Niels Möller | 7d76a31 | 2018-10-26 10:57:07 | [diff] [blame] | 105 | RtpTransportControllerSendInterface* rtp_transport, |
Niels Möller | 67b011d | 2018-10-22 11:00:40 | [diff] [blame] | 106 | BitrateAllocatorInterface* bitrate_allocator, |
michaelt | 9332b7d | 2016-11-30 15:51:13 | [diff] [blame] | 107 | RtcEventLog* event_log, |
ossu | c3d4b48 | 2017-05-23 13:07:11 | [diff] [blame] | 108 | RtcpRttStats* rtcp_rtt_stats, |
Jonas Oreland | a943e73 | 2022-03-16 12:50:58 | [diff] [blame^] | 109 | const absl::optional<RtpState>& suspended_rtp_state, |
| 110 | const WebRtcKeyValueConfig& field_trials) |
| 111 | : AudioSendStream( |
| 112 | clock, |
| 113 | config, |
| 114 | audio_state, |
| 115 | task_queue_factory, |
| 116 | rtp_transport, |
| 117 | bitrate_allocator, |
| 118 | event_log, |
| 119 | suspended_rtp_state, |
| 120 | voe::CreateChannelSend(clock, |
| 121 | task_queue_factory, |
| 122 | config.send_transport, |
| 123 | rtcp_rtt_stats, |
| 124 | event_log, |
| 125 | config.frame_encryptor, |
| 126 | config.crypto_options, |
| 127 | config.rtp.extmap_allow_mixed, |
| 128 | config.rtcp_report_interval_ms, |
| 129 | config.rtp.ssrc, |
| 130 | config.frame_transformer, |
| 131 | rtp_transport->transport_feedback_observer(), |
| 132 | field_trials), |
| 133 | field_trials) {} |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 134 | |
| 135 | AudioSendStream::AudioSendStream( |
Sebastian Jansson | 977b335 | 2019-03-04 16:43:34 | [diff] [blame] | 136 | Clock* clock, |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 137 | const webrtc::AudioSendStream::Config& config, |
| 138 | const rtc::scoped_refptr<webrtc::AudioState>& audio_state, |
Sebastian Jansson | 44dd9f2 | 2019-03-08 13:50:30 | [diff] [blame] | 139 | TaskQueueFactory* task_queue_factory, |
Niels Möller | 7d76a31 | 2018-10-26 10:57:07 | [diff] [blame] | 140 | RtpTransportControllerSendInterface* rtp_transport, |
Niels Möller | 67b011d | 2018-10-22 11:00:40 | [diff] [blame] | 141 | BitrateAllocatorInterface* bitrate_allocator, |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 142 | RtcEventLog* event_log, |
Danil Chapovalov | b9b146c | 2018-06-15 10:28:07 | [diff] [blame] | 143 | const absl::optional<RtpState>& suspended_rtp_state, |
Jonas Oreland | a943e73 | 2022-03-16 12:50:58 | [diff] [blame^] | 144 | std::unique_ptr<voe::ChannelSendInterface> channel_send, |
| 145 | const WebRtcKeyValueConfig& field_trials) |
Sebastian Jansson | 977b335 | 2019-03-04 16:43:34 | [diff] [blame] | 146 | : clock_(clock), |
Jonas Oreland | a943e73 | 2022-03-16 12:50:58 | [diff] [blame^] | 147 | field_trials_(field_trials), |
Markus Handell | 3907e7b | 2021-06-01 07:07:20 | [diff] [blame] | 148 | rtp_transport_queue_(rtp_transport->GetWorkerQueue()), |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 149 | allocate_audio_without_feedback_( |
Jonas Oreland | a943e73 | 2022-03-16 12:50:58 | [diff] [blame^] | 150 | field_trials_.IsEnabled("WebRTC-Audio-ABWENoTWCC")), |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 151 | enable_audio_alr_probing_( |
Jonas Oreland | a943e73 | 2022-03-16 12:50:58 | [diff] [blame^] | 152 | !field_trials_.IsDisabled("WebRTC-Audio-AlrProbing")), |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 153 | send_side_bwe_with_overhead_( |
Jonas Oreland | a943e73 | 2022-03-16 12:50:58 | [diff] [blame^] | 154 | !field_trials_.IsDisabled("WebRTC-SendSideBwe-WithOverhead")), |
| 155 | allocation_settings_(field_trials_), |
Bjorn A Mellem | 7a9a092 | 2019-11-26 17:19:40 | [diff] [blame] | 156 | config_(Config(/*send_transport=*/nullptr)), |
mflodman | 86cc6ff | 2016-07-26 11:44:06 | [diff] [blame] | 157 | audio_state_(audio_state), |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 158 | channel_send_(std::move(channel_send)), |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 159 | event_log_(event_log), |
Sebastian Jansson | 62aee93 | 2019-10-02 10:27:06 | [diff] [blame] | 160 | use_legacy_overhead_calculation_( |
Jonas Oreland | a943e73 | 2022-03-16 12:50:58 | [diff] [blame^] | 161 | field_trials_.IsEnabled("WebRTC-Audio-LegacyOverhead")), |
michaelt | f4caaab | 2017-01-17 07:55:07 | [diff] [blame] | 162 | bitrate_allocator_(bitrate_allocator), |
Niels Möller | 7d76a31 | 2018-10-26 10:57:07 | [diff] [blame] | 163 | rtp_transport_(rtp_transport), |
Sebastian Jansson | 6298b56 | 2020-01-14 16:55:19 | [diff] [blame] | 164 | rtp_rtcp_module_(channel_send_->GetRtpRtcp()), |
Sam Zackrisson | ff05816 | 2018-11-20 16:15:13 | [diff] [blame] | 165 | suspended_rtp_state_(suspended_rtp_state) { |
Jonas Olsson | 24ea822 | 2018-01-25 09:14:29 | [diff] [blame] | 166 | RTC_LOG(LS_INFO) << "AudioSendStream: " << config.rtp.ssrc; |
Markus Handell | 3907e7b | 2021-06-01 07:07:20 | [diff] [blame] | 167 | RTC_DCHECK(rtp_transport_queue_); |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 168 | RTC_DCHECK(audio_state_); |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 169 | RTC_DCHECK(channel_send_); |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 170 | RTC_DCHECK(bitrate_allocator_); |
Sebastian Jansson | 0b69826 | 2019-03-07 08:17:19 | [diff] [blame] | 171 | RTC_DCHECK(rtp_transport); |
| 172 | |
ossu | c3d4b48 | 2017-05-23 13:07:11 | [diff] [blame] | 173 | RTC_DCHECK(rtp_rtcp_module_); |
mflodman | 3d7db26 | 2016-04-29 07:57:13 | [diff] [blame] | 174 | |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 175 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 176 | ConfigureStream(config, true); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 177 | UpdateCachedTargetAudioBitrateConstraints(); |
Sebastian Jansson | c01367d | 2019-04-08 13:20:44 | [diff] [blame] | 178 | pacer_thread_checker_.Detach(); |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 179 | } |
| 180 | |
| 181 | AudioSendStream::~AudioSendStream() { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 182 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Jonas Olsson | 24ea822 | 2018-01-25 09:14:29 | [diff] [blame] | 183 | RTC_LOG(LS_INFO) << "~AudioSendStream: " << config_.rtp.ssrc; |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 184 | RTC_DCHECK(!sending_); |
Sebastian Jansson | 0a6510d | 2019-10-04 07:31:08 | [diff] [blame] | 185 | channel_send_->ResetSenderCongestionControlObjects(); |
Sebastian Jansson | 8672cac | 2019-03-01 14:57:55 | [diff] [blame] | 186 | // Blocking call to synchronize state with worker queue to ensure that there |
| 187 | // are no pending tasks left that keeps references to audio. |
| 188 | rtc::Event thread_sync_event; |
Markus Handell | 3907e7b | 2021-06-01 07:07:20 | [diff] [blame] | 189 | rtp_transport_queue_->PostTask([&] { thread_sync_event.Set(); }); |
Sebastian Jansson | 8672cac | 2019-03-01 14:57:55 | [diff] [blame] | 190 | thread_sync_event.Wait(rtc::Event::kForever); |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 191 | } |
| 192 | |
eladalon | abbc430 | 2017-07-26 09:09:44 | [diff] [blame] | 193 | const webrtc::AudioSendStream::Config& AudioSendStream::GetConfig() const { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 194 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
eladalon | abbc430 | 2017-07-26 09:09:44 | [diff] [blame] | 195 | return config_; |
| 196 | } |
| 197 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 198 | void AudioSendStream::Reconfigure( |
| 199 | const webrtc::AudioSendStream::Config& new_config) { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 200 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 201 | ConfigureStream(new_config, false); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 202 | } |
| 203 | |
Alex Narest | cedd351 | 2017-12-07 19:54:55 | [diff] [blame] | 204 | AudioSendStream::ExtensionIds AudioSendStream::FindExtensionIds( |
| 205 | const std::vector<RtpExtension>& extensions) { |
| 206 | ExtensionIds ids; |
| 207 | for (const auto& extension : extensions) { |
| 208 | if (extension.uri == RtpExtension::kAudioLevelUri) { |
| 209 | ids.audio_level = extension.id; |
Sebastian Jansson | 71c6b56 | 2019-08-14 09:31:02 | [diff] [blame] | 210 | } else if (extension.uri == RtpExtension::kAbsSendTimeUri) { |
| 211 | ids.abs_send_time = extension.id; |
Alex Narest | cedd351 | 2017-12-07 19:54:55 | [diff] [blame] | 212 | } else if (extension.uri == RtpExtension::kTransportSequenceNumberUri) { |
| 213 | ids.transport_sequence_number = extension.id; |
Steve Anton | bb50ce5 | 2018-03-26 17:24:32 | [diff] [blame] | 214 | } else if (extension.uri == RtpExtension::kMidUri) { |
| 215 | ids.mid = extension.id; |
Amit Hilbuch | 77938e6 | 2018-12-21 17:23:38 | [diff] [blame] | 216 | } else if (extension.uri == RtpExtension::kRidUri) { |
| 217 | ids.rid = extension.id; |
| 218 | } else if (extension.uri == RtpExtension::kRepairedRidUri) { |
| 219 | ids.repaired_rid = extension.id; |
Minyue Li | 74dadc1 | 2020-03-05 10:33:13 | [diff] [blame] | 220 | } else if (extension.uri == RtpExtension::kAbsoluteCaptureTimeUri) { |
| 221 | ids.abs_capture_time = extension.id; |
Alex Narest | cedd351 | 2017-12-07 19:54:55 | [diff] [blame] | 222 | } |
| 223 | } |
| 224 | return ids; |
| 225 | } |
| 226 | |
Sebastian Jansson | 470a5ea | 2019-01-23 11:37:49 | [diff] [blame] | 227 | int AudioSendStream::TransportSeqNumId(const AudioSendStream::Config& config) { |
| 228 | return FindExtensionIds(config.rtp.extensions).transport_sequence_number; |
| 229 | } |
| 230 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 231 | void AudioSendStream::ConfigureStream( |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 232 | const webrtc::AudioSendStream::Config& new_config, |
| 233 | bool first_time) { |
Jonas Olsson | 24ea822 | 2018-01-25 09:14:29 | [diff] [blame] | 234 | RTC_LOG(LS_INFO) << "AudioSendStream::ConfigureStream: " |
| 235 | << new_config.ToString(); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 236 | UpdateEventLogStreamConfig(event_log_, new_config, |
| 237 | first_time ? nullptr : &config_); |
Oskar Sundbom | 56ef305 | 2018-10-30 15:11:02 | [diff] [blame] | 238 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 239 | const auto& old_config = config_; |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 240 | |
Niels Möller | e977199 | 2018-11-26 09:55:07 | [diff] [blame] | 241 | // Configuration parameters which cannot be changed. |
| 242 | RTC_DCHECK(first_time || |
| 243 | old_config.send_transport == new_config.send_transport); |
Erik Språng | 70efdde | 2019-08-21 11:36:20 | [diff] [blame] | 244 | RTC_DCHECK(first_time || old_config.rtp.ssrc == new_config.rtp.ssrc); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 245 | if (suspended_rtp_state_ && first_time) { |
| 246 | rtp_rtcp_module_->SetRtpState(*suspended_rtp_state_); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 247 | } |
| 248 | if (first_time || old_config.rtp.c_name != new_config.rtp.c_name) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 249 | channel_send_->SetRTCP_CNAME(new_config.rtp.c_name); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 250 | } |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 251 | |
Benjamin Wright | 84583f6 | 2018-10-04 21:22:34 | [diff] [blame] | 252 | // Enable the frame encryptor if a new frame encryptor has been provided. |
| 253 | if (first_time || new_config.frame_encryptor != old_config.frame_encryptor) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 254 | channel_send_->SetFrameEncryptor(new_config.frame_encryptor); |
Benjamin Wright | 84583f6 | 2018-10-04 21:22:34 | [diff] [blame] | 255 | } |
| 256 | |
Johannes Kron | 9190b82 | 2018-10-29 10:22:05 | [diff] [blame] | 257 | if (first_time || |
Marina Ciocea | d2aa8f9 | 2020-03-31 09:29:56 | [diff] [blame] | 258 | new_config.frame_transformer != old_config.frame_transformer) { |
| 259 | channel_send_->SetEncoderToPacketizerFrameTransformer( |
| 260 | new_config.frame_transformer); |
| 261 | } |
| 262 | |
| 263 | if (first_time || |
Johannes Kron | 9190b82 | 2018-10-29 10:22:05 | [diff] [blame] | 264 | new_config.rtp.extmap_allow_mixed != old_config.rtp.extmap_allow_mixed) { |
Sebastian Jansson | 6298b56 | 2020-01-14 16:55:19 | [diff] [blame] | 265 | rtp_rtcp_module_->SetExtmapAllowMixed(new_config.rtp.extmap_allow_mixed); |
Johannes Kron | 9190b82 | 2018-10-29 10:22:05 | [diff] [blame] | 266 | } |
| 267 | |
Alex Narest | cedd351 | 2017-12-07 19:54:55 | [diff] [blame] | 268 | const ExtensionIds old_ids = FindExtensionIds(old_config.rtp.extensions); |
| 269 | const ExtensionIds new_ids = FindExtensionIds(new_config.rtp.extensions); |
Yves Gerey | 1704801 | 2019-07-26 15:49:52 | [diff] [blame] | 270 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 271 | // Audio level indication |
| 272 | if (first_time || new_ids.audio_level != old_ids.audio_level) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 273 | channel_send_->SetSendAudioLevelIndicationStatus(new_ids.audio_level != 0, |
| 274 | new_ids.audio_level); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 275 | } |
Sebastian Jansson | 71c6b56 | 2019-08-14 09:31:02 | [diff] [blame] | 276 | |
| 277 | if (first_time || new_ids.abs_send_time != old_ids.abs_send_time) { |
Danil Chapovalov | 723b35f | 2021-10-07 17:12:32 | [diff] [blame] | 278 | absl::string_view uri = AbsoluteSendTime::Uri(); |
| 279 | rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(uri); |
Sebastian Jansson | 71c6b56 | 2019-08-14 09:31:02 | [diff] [blame] | 280 | if (new_ids.abs_send_time) { |
Danil Chapovalov | 723b35f | 2021-10-07 17:12:32 | [diff] [blame] | 281 | rtp_rtcp_module_->RegisterRtpHeaderExtension(uri, new_ids.abs_send_time); |
Sebastian Jansson | 71c6b56 | 2019-08-14 09:31:02 | [diff] [blame] | 282 | } |
| 283 | } |
| 284 | |
Sebastian Jansson | 8d9c540 | 2017-11-15 16:22:16 | [diff] [blame] | 285 | bool transport_seq_num_id_changed = |
| 286 | new_ids.transport_sequence_number != old_ids.transport_sequence_number; |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 287 | if (first_time || |
| 288 | (transport_seq_num_id_changed && !allocate_audio_without_feedback_)) { |
ossu | 1129df2 | 2017-06-30 08:38:56 | [diff] [blame] | 289 | if (!first_time) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 290 | channel_send_->ResetSenderCongestionControlObjects(); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 291 | } |
| 292 | |
Sebastian Jansson | 8d9c540 | 2017-11-15 16:22:16 | [diff] [blame] | 293 | RtcpBandwidthObserver* bandwidth_observer = nullptr; |
Sebastian Jansson | 470a5ea | 2019-01-23 11:37:49 | [diff] [blame] | 294 | |
Jakob Ivarsson | 47a03e8 | 2020-11-23 14:05:44 | [diff] [blame] | 295 | if (!allocate_audio_without_feedback_ && |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 296 | new_ids.transport_sequence_number != 0) { |
Sebastian Jansson | 6298b56 | 2020-01-14 16:55:19 | [diff] [blame] | 297 | rtp_rtcp_module_->RegisterRtpHeaderExtension( |
Danil Chapovalov | d0321c5 | 2021-09-14 10:58:51 | [diff] [blame] | 298 | TransportSequenceNumber::Uri(), new_ids.transport_sequence_number); |
Sebastian Jansson | 8d9c540 | 2017-11-15 16:22:16 | [diff] [blame] | 299 | // Probing in application limited region is only used in combination with |
| 300 | // send side congestion control, wich depends on feedback packets which |
| 301 | // requires transport sequence numbers to be enabled. |
Sebastian Jansson | 0a6510d | 2019-10-04 07:31:08 | [diff] [blame] | 302 | // Optionally request ALR probing but do not override any existing |
| 303 | // request from other streams. |
| 304 | if (enable_audio_alr_probing_) { |
| 305 | rtp_transport_->EnablePeriodicAlrProbing(true); |
Niels Möller | 7d76a31 | 2018-10-26 10:57:07 | [diff] [blame] | 306 | } |
Sebastian Jansson | 0a6510d | 2019-10-04 07:31:08 | [diff] [blame] | 307 | bandwidth_observer = rtp_transport_->GetBandwidthObserver(); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 308 | } |
Sebastian Jansson | 0a6510d | 2019-10-04 07:31:08 | [diff] [blame] | 309 | channel_send_->RegisterSenderCongestionControlObjects(rtp_transport_, |
| 310 | bandwidth_observer); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 311 | } |
Steve Anton | bb50ce5 | 2018-03-26 17:24:32 | [diff] [blame] | 312 | // MID RTP header extension. |
Steve Anton | 003930a | 2018-03-29 19:37:21 | [diff] [blame] | 313 | if ((first_time || new_ids.mid != old_ids.mid || |
| 314 | new_config.rtp.mid != old_config.rtp.mid) && |
| 315 | new_ids.mid != 0 && !new_config.rtp.mid.empty()) { |
Danil Chapovalov | d0321c5 | 2021-09-14 10:58:51 | [diff] [blame] | 316 | rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpMid::Uri(), new_ids.mid); |
Sebastian Jansson | 6298b56 | 2020-01-14 16:55:19 | [diff] [blame] | 317 | rtp_rtcp_module_->SetMid(new_config.rtp.mid); |
Steve Anton | bb50ce5 | 2018-03-26 17:24:32 | [diff] [blame] | 318 | } |
| 319 | |
Minyue Li | 74dadc1 | 2020-03-05 10:33:13 | [diff] [blame] | 320 | if (first_time || new_ids.abs_capture_time != old_ids.abs_capture_time) { |
Danil Chapovalov | 723b35f | 2021-10-07 17:12:32 | [diff] [blame] | 321 | absl::string_view uri = AbsoluteCaptureTimeExtension::Uri(); |
| 322 | rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(uri); |
Minyue Li | 74dadc1 | 2020-03-05 10:33:13 | [diff] [blame] | 323 | if (new_ids.abs_capture_time) { |
Danil Chapovalov | 723b35f | 2021-10-07 17:12:32 | [diff] [blame] | 324 | rtp_rtcp_module_->RegisterRtpHeaderExtension(uri, |
| 325 | new_ids.abs_capture_time); |
Minyue Li | 74dadc1 | 2020-03-05 10:33:13 | [diff] [blame] | 326 | } |
| 327 | } |
| 328 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 329 | if (!ReconfigureSendCodec(new_config)) { |
Mirko Bonadei | 675513b | 2017-11-09 10:09:25 | [diff] [blame] | 330 | RTC_LOG(LS_ERROR) << "Failed to set up send codec state."; |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 331 | } |
| 332 | |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 333 | // Set currently known overhead (used in ANA, opus only). |
| 334 | { |
Markus Handell | 6287280 | 2020-07-06 13:15:07 | [diff] [blame] | 335 | MutexLock lock(&overhead_per_packet_lock_); |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 336 | UpdateOverheadForEncoder(); |
| 337 | } |
| 338 | |
Jakob Ivarsson | d14525e | 2020-03-06 08:49:29 | [diff] [blame] | 339 | channel_send_->CallEncoder([this](AudioEncoder* encoder) { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 340 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Jakob Ivarsson | d14525e | 2020-03-06 08:49:29 | [diff] [blame] | 341 | if (!encoder) { |
| 342 | return; |
| 343 | } |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 344 | frame_length_range_ = encoder->GetFrameLengthRange(); |
| 345 | UpdateCachedTargetAudioBitrateConstraints(); |
Jakob Ivarsson | d14525e | 2020-03-06 08:49:29 | [diff] [blame] | 346 | }); |
| 347 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 348 | if (sending_) { |
| 349 | ReconfigureBitrateObserver(new_config); |
Oskar Sundbom | f85e31b | 2017-12-20 15:38:09 | [diff] [blame] | 350 | } |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 351 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 352 | config_ = new_config; |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 353 | if (!first_time) { |
| 354 | UpdateCachedTargetAudioBitrateConstraints(); |
| 355 | } |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 356 | } |
| 357 | |
solenberg | 3a94154 | 2015-11-16 15:34:50 | [diff] [blame] | 358 | void AudioSendStream::Start() { |
Sebastian Jansson | 8672cac | 2019-03-01 14:57:55 | [diff] [blame] | 359 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 360 | if (sending_) { |
| 361 | return; |
| 362 | } |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 363 | if (!config_.has_dscp && config_.min_bitrate_bps != -1 && |
| 364 | config_.max_bitrate_bps != -1 && |
Sebastian Jansson | cd0eedb | 2019-10-10 11:52:26 | [diff] [blame] | 365 | (allocate_audio_without_feedback_ || TransportSeqNumId(config_) != 0)) { |
Erik Språng | aa59eca | 2019-07-24 12:52:55 | [diff] [blame] | 366 | rtp_transport_->AccountForAudioPacketsInPacedSender(true); |
Sebastian Jansson | c3eb9fd | 2020-01-29 16:42:52 | [diff] [blame] | 367 | if (send_side_bwe_with_overhead_) |
| 368 | rtp_transport_->IncludeOverheadInPacedSender(); |
Sebastian Jansson | b686396 | 2018-10-10 08:23:13 | [diff] [blame] | 369 | rtp_rtcp_module_->SetAsPartOfAllocation(true); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 370 | ConfigureBitrateObserver(); |
Sebastian Jansson | b686396 | 2018-10-10 08:23:13 | [diff] [blame] | 371 | } else { |
| 372 | rtp_rtcp_module_->SetAsPartOfAllocation(false); |
mflodman | 86cc6ff | 2016-07-26 11:44:06 | [diff] [blame] | 373 | } |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 374 | channel_send_->StartSend(); |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 375 | sending_ = true; |
| 376 | audio_state()->AddSendingStream(this, encoder_sample_rate_hz_, |
| 377 | encoder_num_channels_); |
solenberg | 3a94154 | 2015-11-16 15:34:50 | [diff] [blame] | 378 | } |
| 379 | |
| 380 | void AudioSendStream::Stop() { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 381 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 382 | if (!sending_) { |
| 383 | return; |
| 384 | } |
| 385 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 386 | RemoveBitrateObserver(); |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 387 | channel_send_->StopSend(); |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 388 | sending_ = false; |
| 389 | audio_state()->RemoveSendingStream(this); |
| 390 | } |
| 391 | |
| 392 | void AudioSendStream::SendAudioData(std::unique_ptr<AudioFrame> audio_frame) { |
| 393 | RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_); |
Henrik Boström | d2c336f | 2019-07-03 15:11:10 | [diff] [blame] | 394 | RTC_DCHECK_GT(audio_frame->sample_rate_hz_, 0); |
| 395 | double duration = static_cast<double>(audio_frame->samples_per_channel_) / |
| 396 | audio_frame->sample_rate_hz_; |
| 397 | { |
| 398 | // Note: SendAudioData() passes the frame further down the pipeline and it |
| 399 | // may eventually get sent. But this method is invoked even if we are not |
| 400 | // connected, as long as we have an AudioSendStream (created as a result of |
| 401 | // an O/A exchange). This means that we are calculating audio levels whether |
| 402 | // or not we are sending samples. |
| 403 | // TODO(https://crbug.com/webrtc/10771): All "media-source" related stats |
| 404 | // should move from send-streams to the local audio sources or tracks; a |
| 405 | // send-stream should not be required to read the microphone audio levels. |
Markus Handell | 6287280 | 2020-07-06 13:15:07 | [diff] [blame] | 406 | MutexLock lock(&audio_level_lock_); |
Henrik Boström | d2c336f | 2019-07-03 15:11:10 | [diff] [blame] | 407 | audio_level_.ComputeLevel(*audio_frame, duration); |
| 408 | } |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 409 | channel_send_->ProcessAndEncodeAudio(std::move(audio_frame)); |
solenberg | 3a94154 | 2015-11-16 15:34:50 | [diff] [blame] | 410 | } |
| 411 | |
solenberg | ffbbcac | 2016-11-17 13:25:37 | [diff] [blame] | 412 | bool AudioSendStream::SendTelephoneEvent(int payload_type, |
Yves Gerey | 665174f | 2018-06-19 13:03:05 | [diff] [blame] | 413 | int payload_frequency, |
| 414 | int event, |
solenberg | 8842c3e | 2016-03-11 11:06:41 | [diff] [blame] | 415 | int duration_ms) { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 416 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Niels Möller | 8fb1a6a | 2019-03-05 13:29:42 | [diff] [blame] | 417 | channel_send_->SetSendTelephoneEventPayloadType(payload_type, |
| 418 | payload_frequency); |
| 419 | return channel_send_->SendTelephoneEventOutband(event, duration_ms); |
Fredrik Solenberg | b572768 | 2015-12-04 14:22:19 | [diff] [blame] | 420 | } |
| 421 | |
solenberg | 9421853 | 2016-06-16 17:53:22 | [diff] [blame] | 422 | void AudioSendStream::SetMuted(bool muted) { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 423 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 424 | channel_send_->SetInputMute(muted); |
solenberg | 9421853 | 2016-06-16 17:53:22 | [diff] [blame] | 425 | } |
| 426 | |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 427 | webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const { |
Ivo Creusen | 56d46090 | 2017-11-24 16:29:59 | [diff] [blame] | 428 | return GetStats(true); |
| 429 | } |
| 430 | |
| 431 | webrtc::AudioSendStream::Stats AudioSendStream::GetStats( |
| 432 | bool has_remote_tracks) const { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 433 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 434 | webrtc::AudioSendStream::Stats stats; |
| 435 | stats.local_ssrc = config_.rtp.ssrc; |
Jakob Ivarsson | bf08745 | 2021-11-11 12:43:49 | [diff] [blame] | 436 | stats.target_bitrate_bps = channel_send_->GetTargetBitrate(); |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 437 | |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 438 | webrtc::CallSendStatistics call_stats = channel_send_->GetRTCPStatistics(); |
Niels Möller | ac0a4cb | 2019-10-09 13:01:33 | [diff] [blame] | 439 | stats.payload_bytes_sent = call_stats.payload_bytes_sent; |
| 440 | stats.header_and_padding_bytes_sent = |
| 441 | call_stats.header_and_padding_bytes_sent; |
Henrik Boström | cf96e0f | 2019-04-17 11:51:53 | [diff] [blame] | 442 | stats.retransmitted_bytes_sent = call_stats.retransmitted_bytes_sent; |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 443 | stats.packets_sent = call_stats.packetsSent; |
Henrik Boström | cf96e0f | 2019-04-17 11:51:53 | [diff] [blame] | 444 | stats.retransmitted_packets_sent = call_stats.retransmitted_packets_sent; |
solenberg | 8b85de2 | 2015-11-16 17:48:04 | [diff] [blame] | 445 | // RTT isn't known until a RTCP report is received. Until then, VoiceEngine |
| 446 | // returns 0 to indicate an error value. |
| 447 | if (call_stats.rttMs > 0) { |
| 448 | stats.rtt_ms = call_stats.rttMs; |
| 449 | } |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 450 | if (config_.send_codec_spec) { |
| 451 | const auto& spec = *config_.send_codec_spec; |
| 452 | stats.codec_name = spec.format.name; |
Oskar Sundbom | 2707fb2 | 2017-11-16 09:57:35 | [diff] [blame] | 453 | stats.codec_payload_type = spec.payload_type; |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 454 | |
| 455 | // Get data from the last remote RTCP report. |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 456 | for (const auto& block : channel_send_->GetRemoteRTCPReportBlocks()) { |
solenberg | 8b85de2 | 2015-11-16 17:48:04 | [diff] [blame] | 457 | // Lookup report for send ssrc only. |
| 458 | if (block.source_SSRC == stats.local_ssrc) { |
| 459 | stats.packets_lost = block.cumulative_num_packets_lost; |
| 460 | stats.fraction_lost = Q8ToFloat(block.fraction_lost); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 461 | // Convert timestamps to milliseconds. |
| 462 | if (spec.format.clockrate_hz / 1000 > 0) { |
solenberg | 8b85de2 | 2015-11-16 17:48:04 | [diff] [blame] | 463 | stats.jitter_ms = |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 464 | block.interarrival_jitter / (spec.format.clockrate_hz / 1000); |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 465 | } |
solenberg | 8b85de2 | 2015-11-16 17:48:04 | [diff] [blame] | 466 | break; |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 467 | } |
| 468 | } |
| 469 | } |
| 470 | |
Henrik Boström | d2c336f | 2019-07-03 15:11:10 | [diff] [blame] | 471 | { |
Markus Handell | 6287280 | 2020-07-06 13:15:07 | [diff] [blame] | 472 | MutexLock lock(&audio_level_lock_); |
Henrik Boström | d2c336f | 2019-07-03 15:11:10 | [diff] [blame] | 473 | stats.audio_level = audio_level_.LevelFullRange(); |
| 474 | stats.total_input_energy = audio_level_.TotalEnergy(); |
| 475 | stats.total_input_duration = audio_level_.TotalDuration(); |
| 476 | } |
solenberg | 796b8f9 | 2017-03-02 01:02:23 | [diff] [blame] | 477 | |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 478 | stats.typing_noise_detected = audio_state()->typing_noise_detected(); |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 479 | stats.ana_statistics = channel_send_->GetANAStatistics(); |
Per Åhgren | cc73ed3 | 2020-04-26 21:56:17 | [diff] [blame] | 480 | |
| 481 | AudioProcessing* ap = audio_state_->audio_processing(); |
| 482 | if (ap) { |
| 483 | stats.apm_statistics = ap->GetStatistics(has_remote_tracks); |
| 484 | } |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 485 | |
Henrik Boström | 6e436d1 | 2019-05-27 10:19:33 | [diff] [blame] | 486 | stats.report_block_datas = std::move(call_stats.report_block_datas); |
| 487 | |
Jakob Ivarsson | e91c992 | 2021-07-06 07:55:43 | [diff] [blame] | 488 | stats.nacks_rcvd = call_stats.nacks_rcvd; |
| 489 | |
solenberg | 85a0496 | 2015-10-27 10:35:21 | [diff] [blame] | 490 | return stats; |
| 491 | } |
| 492 | |
Niels Möller | 8fb1a6a | 2019-03-05 13:29:42 | [diff] [blame] | 493 | void AudioSendStream::DeliverRtcp(const uint8_t* packet, size_t length) { |
Erik Språng | 2b4d2f3 | 2020-06-29 14:37:44 | [diff] [blame] | 494 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Niels Möller | 8fb1a6a | 2019-03-05 13:29:42 | [diff] [blame] | 495 | channel_send_->ReceivedRTCPPacket(packet, length); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 496 | |
| 497 | { |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 498 | // Poll if overhead has changed, which it can do if ack triggers us to stop |
| 499 | // sending mid/rid. |
Markus Handell | 6287280 | 2020-07-06 13:15:07 | [diff] [blame] | 500 | MutexLock lock(&overhead_per_packet_lock_); |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 501 | UpdateOverheadForEncoder(); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 502 | } |
| 503 | UpdateCachedTargetAudioBitrateConstraints(); |
pbos | 1ba8d39 | 2016-05-02 03:18:34 | [diff] [blame] | 504 | } |
| 505 | |
Sebastian Jansson | c0e4d45 | 2018-10-25 13:08:32 | [diff] [blame] | 506 | uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) { |
Markus Handell | 3907e7b | 2021-06-01 07:07:20 | [diff] [blame] | 507 | RTC_DCHECK_RUN_ON(rtp_transport_queue_); |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 508 | |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 509 | // Pick a target bitrate between the constraints. Overrules the allocator if |
| 510 | // it 1) allocated a bitrate of zero to disable the stream or 2) allocated a |
| 511 | // higher than max to allow for e.g. extra FEC. |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 512 | RTC_DCHECK(cached_constraints_.has_value()); |
| 513 | update.target_bitrate.Clamp(cached_constraints_->min, |
| 514 | cached_constraints_->max); |
| 515 | update.stable_target_bitrate.Clamp(cached_constraints_->min, |
| 516 | cached_constraints_->max); |
mflodman | 86cc6ff | 2016-07-26 11:44:06 | [diff] [blame] | 517 | |
Sebastian Jansson | 254d869 | 2018-11-21 18:19:00 | [diff] [blame] | 518 | channel_send_->OnBitrateAllocation(update); |
mflodman | 86cc6ff | 2016-07-26 11:44:06 | [diff] [blame] | 519 | |
| 520 | // The amount of audio protection is not exposed by the encoder, hence |
| 521 | // always returning 0. |
| 522 | return 0; |
| 523 | } |
| 524 | |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 525 | void AudioSendStream::SetTransportOverhead( |
| 526 | int transport_overhead_per_packet_bytes) { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 527 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
| 528 | { |
| 529 | MutexLock lock(&overhead_per_packet_lock_); |
| 530 | transport_overhead_per_packet_bytes_ = transport_overhead_per_packet_bytes; |
| 531 | UpdateOverheadForEncoder(); |
| 532 | } |
| 533 | UpdateCachedTargetAudioBitrateConstraints(); |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 534 | } |
| 535 | |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 536 | void AudioSendStream::UpdateOverheadForEncoder() { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 537 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Erik Språng | cf6544a | 2020-05-13 12:43:11 | [diff] [blame] | 538 | size_t overhead_per_packet_bytes = GetPerPacketOverheadBytes(); |
| 539 | if (overhead_per_packet_ == overhead_per_packet_bytes) { |
| 540 | return; |
Bjorn A Mellem | 413ccc4 | 2019-04-26 22:41:05 | [diff] [blame] | 541 | } |
Erik Språng | cf6544a | 2020-05-13 12:43:11 | [diff] [blame] | 542 | overhead_per_packet_ = overhead_per_packet_bytes; |
| 543 | |
Sebastian Jansson | 14a7cf9 | 2019-02-13 14:11:42 | [diff] [blame] | 544 | channel_send_->CallEncoder([&](AudioEncoder* encoder) { |
| 545 | encoder->OnReceivedOverhead(overhead_per_packet_bytes); |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 546 | }); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 547 | if (total_packet_overhead_bytes_ != overhead_per_packet_bytes) { |
| 548 | total_packet_overhead_bytes_ = overhead_per_packet_bytes; |
| 549 | if (registered_with_allocator_) { |
| 550 | ConfigureBitrateObserver(); |
Sebastian Jansson | 8672cac | 2019-03-01 14:57:55 | [diff] [blame] | 551 | } |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 552 | } |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 553 | } |
| 554 | |
| 555 | size_t AudioSendStream::TestOnlyGetPerPacketOverheadBytes() const { |
Markus Handell | 6287280 | 2020-07-06 13:15:07 | [diff] [blame] | 556 | MutexLock lock(&overhead_per_packet_lock_); |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 557 | return GetPerPacketOverheadBytes(); |
| 558 | } |
| 559 | |
| 560 | size_t AudioSendStream::GetPerPacketOverheadBytes() const { |
| 561 | return transport_overhead_per_packet_bytes_ + |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 562 | rtp_rtcp_module_->ExpectedPerPacketOverhead(); |
michaelt | 79e0588 | 2016-11-08 10:50:09 | [diff] [blame] | 563 | } |
| 564 | |
ossu | c3d4b48 | 2017-05-23 13:07:11 | [diff] [blame] | 565 | RtpState AudioSendStream::GetRtpState() const { |
| 566 | return rtp_rtcp_module_->GetRtpState(); |
| 567 | } |
| 568 | |
Niels Möller | dced9f6 | 2018-11-19 09:27:07 | [diff] [blame] | 569 | const voe::ChannelSendInterface* AudioSendStream::GetChannel() const { |
| 570 | return channel_send_.get(); |
Fredrik Solenberg | 8f5787a | 2018-01-11 12:52:30 | [diff] [blame] | 571 | } |
| 572 | |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 573 | internal::AudioState* AudioSendStream::audio_state() { |
| 574 | internal::AudioState* audio_state = |
| 575 | static_cast<internal::AudioState*>(audio_state_.get()); |
| 576 | RTC_DCHECK(audio_state); |
| 577 | return audio_state; |
| 578 | } |
| 579 | |
| 580 | const internal::AudioState* AudioSendStream::audio_state() const { |
| 581 | internal::AudioState* audio_state = |
| 582 | static_cast<internal::AudioState*>(audio_state_.get()); |
| 583 | RTC_DCHECK(audio_state); |
| 584 | return audio_state; |
| 585 | } |
| 586 | |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 587 | void AudioSendStream::StoreEncoderProperties(int sample_rate_hz, |
| 588 | size_t num_channels) { |
Fredrik Solenberg | 2a87797 | 2017-12-15 15:42:15 | [diff] [blame] | 589 | encoder_sample_rate_hz_ = sample_rate_hz; |
| 590 | encoder_num_channels_ = num_channels; |
| 591 | if (sending_) { |
| 592 | // Update AudioState's information about the stream. |
| 593 | audio_state()->AddSendingStream(this, sample_rate_hz, num_channels); |
| 594 | } |
| 595 | } |
| 596 | |
minyue | 7a97344 | 2016-10-20 10:27:12 | [diff] [blame] | 597 | // Apply current codec settings to a single voe::Channel used for sending. |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 598 | bool AudioSendStream::SetupSendCodec(const Config& new_config) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 599 | RTC_DCHECK(new_config.send_codec_spec); |
| 600 | const auto& spec = *new_config.send_codec_spec; |
minyue | 48368ad | 2017-05-10 11:06:11 | [diff] [blame] | 601 | |
| 602 | RTC_DCHECK(new_config.encoder_factory); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 603 | std::unique_ptr<AudioEncoder> encoder = |
Karl Wiberg | 77490b9 | 2018-03-21 14:18:42 | [diff] [blame] | 604 | new_config.encoder_factory->MakeAudioEncoder( |
| 605 | spec.payload_type, spec.format, new_config.codec_pair_id); |
minyue | 7a97344 | 2016-10-20 10:27:12 | [diff] [blame] | 606 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 607 | if (!encoder) { |
Jonas Olsson | abbe841 | 2018-04-03 11:40:05 | [diff] [blame] | 608 | RTC_DLOG(LS_ERROR) << "Unable to create encoder for " |
| 609 | << rtc::ToString(spec.format); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 610 | return false; |
| 611 | } |
Alex Narest | bbbe4e1 | 2018-07-13 08:32:58 | [diff] [blame] | 612 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 613 | // If a bitrate has been specified for the codec, use it over the |
| 614 | // codec's default. |
Christoffer Rodbro | 110c64b | 2019-03-06 08:51:08 | [diff] [blame] | 615 | if (spec.target_bitrate_bps) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 616 | encoder->OnReceivedTargetAudioBitrate(*spec.target_bitrate_bps); |
minyue | 7a97344 | 2016-10-20 10:27:12 | [diff] [blame] | 617 | } |
| 618 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 619 | // Enable ANA if configured (currently only used by Opus). |
Mirko Bonadei | 4356490 | 2020-01-29 15:29:36 | [diff] [blame] | 620 | if (new_config.audio_network_adaptor_config) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 621 | if (encoder->EnableAudioNetworkAdaptor( |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 622 | *new_config.audio_network_adaptor_config, event_log_)) { |
Jakob Ivarsson | ed97116 | 2020-08-11 12:05:07 | [diff] [blame] | 623 | RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC " |
| 624 | << new_config.rtp.ssrc; |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 625 | } else { |
Jakob Ivarsson | ed97116 | 2020-08-11 12:05:07 | [diff] [blame] | 626 | RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC " |
| 627 | << new_config.rtp.ssrc; |
minyue | 6b825df | 2016-10-31 11:08:32 | [diff] [blame] | 628 | } |
minyue | 7a97344 | 2016-10-20 10:27:12 | [diff] [blame] | 629 | } |
| 630 | |
Philipp Hancke | 1a49756 | 2020-05-26 17:12:31 | [diff] [blame] | 631 | // Wrap the encoder in an AudioEncoderCNG, if VAD is enabled. |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 632 | if (spec.cng_payload_type) { |
Karl Wiberg | 2365936 | 2018-11-01 10:13:44 | [diff] [blame] | 633 | AudioEncoderCngConfig cng_config; |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 634 | cng_config.num_channels = encoder->NumChannels(); |
| 635 | cng_config.payload_type = *spec.cng_payload_type; |
| 636 | cng_config.speech_encoder = std::move(encoder); |
| 637 | cng_config.vad_mode = Vad::kVadNormal; |
Karl Wiberg | 2365936 | 2018-11-01 10:13:44 | [diff] [blame] | 638 | encoder = CreateComfortNoiseEncoder(std::move(cng_config)); |
ossu | 3b9ff38 | 2017-04-27 15:03:42 | [diff] [blame] | 639 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 640 | RegisterCngPayloadType(*spec.cng_payload_type, |
| 641 | new_config.send_codec_spec->format.clockrate_hz); |
minyue | 7a97344 | 2016-10-20 10:27:12 | [diff] [blame] | 642 | } |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 643 | |
Philipp Hancke | edcd966 | 2020-06-24 10:52:42 | [diff] [blame] | 644 | // Wrap the encoder in a RED encoder, if RED is enabled. |
| 645 | if (spec.red_payload_type) { |
| 646 | AudioEncoderCopyRed::Config red_config; |
| 647 | red_config.payload_type = *spec.red_payload_type; |
| 648 | red_config.speech_encoder = std::move(encoder); |
Jonas Oreland | a943e73 | 2022-03-16 12:50:58 | [diff] [blame^] | 649 | encoder = std::make_unique<AudioEncoderCopyRed>(std::move(red_config), |
| 650 | field_trials_); |
Philipp Hancke | edcd966 | 2020-06-24 10:52:42 | [diff] [blame] | 651 | } |
| 652 | |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 653 | // Set currently known overhead (used in ANA, opus only). |
| 654 | // If overhead changes later, it will be updated in UpdateOverheadForEncoder. |
| 655 | { |
Markus Handell | 6287280 | 2020-07-06 13:15:07 | [diff] [blame] | 656 | MutexLock lock(&overhead_per_packet_lock_); |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 657 | size_t overhead = GetPerPacketOverheadBytes(); |
| 658 | if (overhead > 0) { |
| 659 | encoder->OnReceivedOverhead(overhead); |
Bjorn A Mellem | 413ccc4 | 2019-04-26 22:41:05 | [diff] [blame] | 660 | } |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 661 | } |
| 662 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 663 | StoreEncoderProperties(encoder->SampleRateHz(), encoder->NumChannels()); |
| 664 | channel_send_->SetEncoder(new_config.send_codec_spec->payload_type, |
| 665 | std::move(encoder)); |
Anton Sukhanov | 626015d | 2019-02-04 23:16:06 | [diff] [blame] | 666 | |
minyue | 7a97344 | 2016-10-20 10:27:12 | [diff] [blame] | 667 | return true; |
| 668 | } |
| 669 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 670 | bool AudioSendStream::ReconfigureSendCodec(const Config& new_config) { |
| 671 | const auto& old_config = config_; |
minyue-webrtc | 8de1826 | 2017-07-26 12:18:40 | [diff] [blame] | 672 | |
| 673 | if (!new_config.send_codec_spec) { |
| 674 | // We cannot de-configure a send codec. So we will do nothing. |
| 675 | // By design, the send codec should have not been configured. |
| 676 | RTC_DCHECK(!old_config.send_codec_spec); |
| 677 | return true; |
| 678 | } |
| 679 | |
| 680 | if (new_config.send_codec_spec == old_config.send_codec_spec && |
| 681 | new_config.audio_network_adaptor_config == |
| 682 | old_config.audio_network_adaptor_config) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 683 | return true; |
| 684 | } |
| 685 | |
| 686 | // If we have no encoder, or the format or payload type's changed, create a |
| 687 | // new encoder. |
| 688 | if (!old_config.send_codec_spec || |
| 689 | new_config.send_codec_spec->format != |
| 690 | old_config.send_codec_spec->format || |
| 691 | new_config.send_codec_spec->payload_type != |
Philipp Hancke | 6144b84 | 2021-06-04 11:49:27 | [diff] [blame] | 692 | old_config.send_codec_spec->payload_type || |
| 693 | new_config.send_codec_spec->red_payload_type != |
| 694 | old_config.send_codec_spec->red_payload_type) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 695 | return SetupSendCodec(new_config); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 696 | } |
| 697 | |
Danil Chapovalov | b9b146c | 2018-06-15 10:28:07 | [diff] [blame] | 698 | const absl::optional<int>& new_target_bitrate_bps = |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 699 | new_config.send_codec_spec->target_bitrate_bps; |
| 700 | // If a bitrate has been specified for the codec, use it over the |
| 701 | // codec's default. |
Christoffer Rodbro | 110c64b | 2019-03-06 08:51:08 | [diff] [blame] | 702 | if (new_target_bitrate_bps && |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 703 | new_target_bitrate_bps != |
| 704 | old_config.send_codec_spec->target_bitrate_bps) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 705 | channel_send_->CallEncoder([&](AudioEncoder* encoder) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 706 | encoder->OnReceivedTargetAudioBitrate(*new_target_bitrate_bps); |
| 707 | }); |
| 708 | } |
| 709 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 710 | ReconfigureANA(new_config); |
| 711 | ReconfigureCNG(new_config); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 712 | |
| 713 | return true; |
| 714 | } |
| 715 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 716 | void AudioSendStream::ReconfigureANA(const Config& new_config) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 717 | if (new_config.audio_network_adaptor_config == |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 718 | config_.audio_network_adaptor_config) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 719 | return; |
| 720 | } |
Mirko Bonadei | 4356490 | 2020-01-29 15:29:36 | [diff] [blame] | 721 | if (new_config.audio_network_adaptor_config) { |
Jakob Ivarsson | fde2b24 | 2020-08-20 14:48:49 | [diff] [blame] | 722 | // This lock needs to be acquired before CallEncoder, since it aquires |
| 723 | // another lock and we need to maintain the same order at all call sites to |
| 724 | // avoid deadlock. |
| 725 | MutexLock lock(&overhead_per_packet_lock_); |
| 726 | size_t overhead = GetPerPacketOverheadBytes(); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 727 | channel_send_->CallEncoder([&](AudioEncoder* encoder) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 728 | if (encoder->EnableAudioNetworkAdaptor( |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 729 | *new_config.audio_network_adaptor_config, event_log_)) { |
Jakob Ivarsson | ed97116 | 2020-08-11 12:05:07 | [diff] [blame] | 730 | RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC " |
| 731 | << new_config.rtp.ssrc; |
Jakob Ivarsson | fde2b24 | 2020-08-20 14:48:49 | [diff] [blame] | 732 | if (overhead > 0) { |
| 733 | encoder->OnReceivedOverhead(overhead); |
| 734 | } |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 735 | } else { |
Jakob Ivarsson | ed97116 | 2020-08-11 12:05:07 | [diff] [blame] | 736 | RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC " |
| 737 | << new_config.rtp.ssrc; |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 738 | } |
| 739 | }); |
| 740 | } else { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 741 | channel_send_->CallEncoder( |
Sebastian Jansson | 14a7cf9 | 2019-02-13 14:11:42 | [diff] [blame] | 742 | [&](AudioEncoder* encoder) { encoder->DisableAudioNetworkAdaptor(); }); |
Jakob Ivarsson | ed97116 | 2020-08-11 12:05:07 | [diff] [blame] | 743 | RTC_LOG(LS_INFO) << "Audio network adaptor disabled on SSRC " |
| 744 | << new_config.rtp.ssrc; |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 745 | } |
| 746 | } |
| 747 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 748 | void AudioSendStream::ReconfigureCNG(const Config& new_config) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 749 | if (new_config.send_codec_spec->cng_payload_type == |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 750 | config_.send_codec_spec->cng_payload_type) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 751 | return; |
| 752 | } |
| 753 | |
ossu | 3b9ff38 | 2017-04-27 15:03:42 | [diff] [blame] | 754 | // Register the CNG payload type if it's been added, don't do anything if CNG |
| 755 | // is removed. Payload types must not be redefined. |
| 756 | if (new_config.send_codec_spec->cng_payload_type) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 757 | RegisterCngPayloadType(*new_config.send_codec_spec->cng_payload_type, |
| 758 | new_config.send_codec_spec->format.clockrate_hz); |
ossu | 3b9ff38 | 2017-04-27 15:03:42 | [diff] [blame] | 759 | } |
| 760 | |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 761 | // Wrap or unwrap the encoder in an AudioEncoderCNG. |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 762 | channel_send_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder_ptr) { |
| 763 | std::unique_ptr<AudioEncoder> old_encoder(std::move(*encoder_ptr)); |
| 764 | auto sub_encoders = old_encoder->ReclaimContainedEncoders(); |
| 765 | if (!sub_encoders.empty()) { |
| 766 | // Replace enc with its sub encoder. We need to put the sub |
| 767 | // encoder in a temporary first, since otherwise the old value |
| 768 | // of enc would be destroyed before the new value got assigned, |
| 769 | // which would be bad since the new value is a part of the old |
| 770 | // value. |
| 771 | auto tmp = std::move(sub_encoders[0]); |
| 772 | old_encoder = std::move(tmp); |
| 773 | } |
| 774 | if (new_config.send_codec_spec->cng_payload_type) { |
| 775 | AudioEncoderCngConfig config; |
| 776 | config.speech_encoder = std::move(old_encoder); |
| 777 | config.num_channels = config.speech_encoder->NumChannels(); |
| 778 | config.payload_type = *new_config.send_codec_spec->cng_payload_type; |
| 779 | config.vad_mode = Vad::kVadNormal; |
| 780 | *encoder_ptr = CreateComfortNoiseEncoder(std::move(config)); |
| 781 | } else { |
| 782 | *encoder_ptr = std::move(old_encoder); |
| 783 | } |
| 784 | }); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 785 | } |
| 786 | |
| 787 | void AudioSendStream::ReconfigureBitrateObserver( |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 788 | const webrtc::AudioSendStream::Config& new_config) { |
| 789 | // Since the Config's default is for both of these to be -1, this test will |
| 790 | // allow us to configure the bitrate observer if the new config has bitrate |
| 791 | // limits set, but would only have us call RemoveBitrateObserver if we were |
| 792 | // previously configured with bitrate limits. |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 793 | if (config_.min_bitrate_bps == new_config.min_bitrate_bps && |
| 794 | config_.max_bitrate_bps == new_config.max_bitrate_bps && |
| 795 | config_.bitrate_priority == new_config.bitrate_priority && |
Jakob Ivarsson | 47a03e8 | 2020-11-23 14:05:44 | [diff] [blame] | 796 | TransportSeqNumId(config_) == TransportSeqNumId(new_config) && |
Jakob Ivarsson | d14525e | 2020-03-06 08:49:29 | [diff] [blame] | 797 | config_.audio_network_adaptor_config == |
| 798 | new_config.audio_network_adaptor_config) { |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 799 | return; |
| 800 | } |
| 801 | |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 802 | if (!new_config.has_dscp && new_config.min_bitrate_bps != -1 && |
Sebastian Jansson | cd0eedb | 2019-10-10 11:52:26 | [diff] [blame] | 803 | new_config.max_bitrate_bps != -1 && TransportSeqNumId(new_config) != 0) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 804 | rtp_transport_->AccountForAudioPacketsInPacedSender(true); |
Sebastian Jansson | c3eb9fd | 2020-01-29 16:42:52 | [diff] [blame] | 805 | if (send_side_bwe_with_overhead_) |
| 806 | rtp_transport_->IncludeOverheadInPacedSender(); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 807 | // We may get a callback immediately as the observer is registered, so |
| 808 | // make sure the bitrate limits in config_ are up-to-date. |
| 809 | config_.min_bitrate_bps = new_config.min_bitrate_bps; |
| 810 | config_.max_bitrate_bps = new_config.max_bitrate_bps; |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 811 | |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 812 | config_.bitrate_priority = new_config.bitrate_priority; |
| 813 | ConfigureBitrateObserver(); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 814 | rtp_rtcp_module_->SetAsPartOfAllocation(true); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 815 | } else { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 07:30:32 | [diff] [blame] | 816 | rtp_transport_->AccountForAudioPacketsInPacedSender(false); |
| 817 | RemoveBitrateObserver(); |
| 818 | rtp_rtcp_module_->SetAsPartOfAllocation(false); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 819 | } |
| 820 | } |
| 821 | |
Sebastian Jansson | 8672cac | 2019-03-01 14:57:55 | [diff] [blame] | 822 | void AudioSendStream::ConfigureBitrateObserver() { |
| 823 | // This either updates the current observer or adds a new observer. |
| 824 | // TODO(srte): Add overhead compensation here. |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 825 | auto constraints = GetMinMaxBitrateConstraints(); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 826 | RTC_DCHECK(constraints.has_value()); |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 827 | |
Sebastian Jansson | 0429f78 | 2019-10-03 16:32:45 | [diff] [blame] | 828 | DataRate priority_bitrate = allocation_settings_.priority_bitrate; |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 829 | if (send_side_bwe_with_overhead_) { |
Sebastian Jansson | 0429f78 | 2019-10-03 16:32:45 | [diff] [blame] | 830 | if (use_legacy_overhead_calculation_) { |
| 831 | // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12) |
| 832 | constexpr int kOverheadPerPacket = 20 + 8 + 10 + 12; |
Danil Chapovalov | 0c626af | 2020-02-10 10:16:00 | [diff] [blame] | 833 | const TimeDelta kMinPacketDuration = TimeDelta::Millis(20); |
Sebastian Jansson | 0429f78 | 2019-10-03 16:32:45 | [diff] [blame] | 834 | DataRate max_overhead = |
Danil Chapovalov | cad3e0e | 2020-02-17 17:46:07 | [diff] [blame] | 835 | DataSize::Bytes(kOverheadPerPacket) / kMinPacketDuration; |
Sebastian Jansson | 0429f78 | 2019-10-03 16:32:45 | [diff] [blame] | 836 | priority_bitrate += max_overhead; |
| 837 | } else { |
| 838 | RTC_DCHECK(frame_length_range_); |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 839 | const DataSize overhead_per_packet = |
Danil Chapovalov | cad3e0e | 2020-02-17 17:46:07 | [diff] [blame] | 840 | DataSize::Bytes(total_packet_overhead_bytes_); |
Erik Språng | 04e1bab | 2020-05-07 16:18:32 | [diff] [blame] | 841 | DataRate min_overhead = overhead_per_packet / frame_length_range_->second; |
Jakob Ivarsson | 01ab084 | 2020-03-06 08:59:56 | [diff] [blame] | 842 | priority_bitrate += min_overhead; |
Sebastian Jansson | 0429f78 | 2019-10-03 16:32:45 | [diff] [blame] | 843 | } |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 844 | } |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 845 | if (allocation_settings_.priority_bitrate_raw) |
| 846 | priority_bitrate = *allocation_settings_.priority_bitrate_raw; |
| 847 | |
Markus Handell | 3907e7b | 2021-06-01 07:07:20 | [diff] [blame] | 848 | rtp_transport_queue_->PostTask([this, constraints, priority_bitrate, |
| 849 | config_bitrate_priority = |
| 850 | config_.bitrate_priority] { |
| 851 | RTC_DCHECK_RUN_ON(rtp_transport_queue_); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 852 | bitrate_allocator_->AddObserver( |
| 853 | this, |
| 854 | MediaStreamAllocationConfig{ |
| 855 | constraints->min.bps<uint32_t>(), constraints->max.bps<uint32_t>(), |
| 856 | 0, priority_bitrate.bps(), true, |
| 857 | allocation_settings_.bitrate_priority.value_or( |
| 858 | config_bitrate_priority)}); |
| 859 | }); |
Jakob Ivarsson | d14525e | 2020-03-06 08:49:29 | [diff] [blame] | 860 | registered_with_allocator_ = true; |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 861 | } |
| 862 | |
| 863 | void AudioSendStream::RemoveBitrateObserver() { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 864 | registered_with_allocator_ = false; |
Niels Möller | c572ff3 | 2018-11-07 07:43:50 | [diff] [blame] | 865 | rtc::Event thread_sync_event; |
Markus Handell | 3907e7b | 2021-06-01 07:07:20 | [diff] [blame] | 866 | rtp_transport_queue_->PostTask([this, &thread_sync_event] { |
| 867 | RTC_DCHECK_RUN_ON(rtp_transport_queue_); |
ossu | 20a4b3f | 2017-04-27 09:08:52 | [diff] [blame] | 868 | bitrate_allocator_->RemoveObserver(this); |
| 869 | thread_sync_event.Set(); |
| 870 | }); |
| 871 | thread_sync_event.Wait(rtc::Event::kForever); |
| 872 | } |
| 873 | |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 874 | absl::optional<AudioSendStream::TargetAudioBitrateConstraints> |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 875 | AudioSendStream::GetMinMaxBitrateConstraints() const { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 876 | if (config_.min_bitrate_bps < 0 || config_.max_bitrate_bps < 0) { |
| 877 | RTC_LOG(LS_WARNING) << "Config is invalid: min_bitrate_bps=" |
| 878 | << config_.min_bitrate_bps |
| 879 | << "; max_bitrate_bps=" << config_.max_bitrate_bps |
| 880 | << "; both expected greater or equal to 0"; |
| 881 | return absl::nullopt; |
| 882 | } |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 883 | TargetAudioBitrateConstraints constraints{ |
Danil Chapovalov | cad3e0e | 2020-02-17 17:46:07 | [diff] [blame] | 884 | DataRate::BitsPerSec(config_.min_bitrate_bps), |
| 885 | DataRate::BitsPerSec(config_.max_bitrate_bps)}; |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 886 | |
| 887 | // If bitrates were explicitly overriden via field trial, use those values. |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 888 | if (allocation_settings_.min_bitrate) |
| 889 | constraints.min = *allocation_settings_.min_bitrate; |
| 890 | if (allocation_settings_.max_bitrate) |
| 891 | constraints.max = *allocation_settings_.max_bitrate; |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 892 | |
Sebastian Jansson | 62aee93 | 2019-10-02 10:27:06 | [diff] [blame] | 893 | RTC_DCHECK_GE(constraints.min, DataRate::Zero()); |
| 894 | RTC_DCHECK_GE(constraints.max, DataRate::Zero()); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 895 | if (constraints.max < constraints.min) { |
| 896 | RTC_LOG(LS_WARNING) << "TargetAudioBitrateConstraints::max is less than " |
| 897 | << "TargetAudioBitrateConstraints::min"; |
| 898 | return absl::nullopt; |
| 899 | } |
Sebastian Jansson | f23131f | 2019-10-03 08:03:55 | [diff] [blame] | 900 | if (send_side_bwe_with_overhead_) { |
Sebastian Jansson | 62aee93 | 2019-10-02 10:27:06 | [diff] [blame] | 901 | if (use_legacy_overhead_calculation_) { |
| 902 | // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12) |
Danil Chapovalov | cad3e0e | 2020-02-17 17:46:07 | [diff] [blame] | 903 | const DataSize kOverheadPerPacket = DataSize::Bytes(20 + 8 + 10 + 12); |
Sebastian Jansson | 62aee93 | 2019-10-02 10:27:06 | [diff] [blame] | 904 | const TimeDelta kMaxFrameLength = |
Danil Chapovalov | 0c626af | 2020-02-10 10:16:00 | [diff] [blame] | 905 | TimeDelta::Millis(60); // Based on Opus spec |
Sebastian Jansson | 62aee93 | 2019-10-02 10:27:06 | [diff] [blame] | 906 | const DataRate kMinOverhead = kOverheadPerPacket / kMaxFrameLength; |
| 907 | constraints.min += kMinOverhead; |
| 908 | constraints.max += kMinOverhead; |
| 909 | } else { |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 910 | if (!frame_length_range_.has_value()) { |
| 911 | RTC_LOG(LS_WARNING) << "frame_length_range_ is not set"; |
| 912 | return absl::nullopt; |
| 913 | } |
Sebastian Jansson | 62aee93 | 2019-10-02 10:27:06 | [diff] [blame] | 914 | const DataSize kOverheadPerPacket = |
Danil Chapovalov | cad3e0e | 2020-02-17 17:46:07 | [diff] [blame] | 915 | DataSize::Bytes(total_packet_overhead_bytes_); |
Sebastian Jansson | 62aee93 | 2019-10-02 10:27:06 | [diff] [blame] | 916 | constraints.min += kOverheadPerPacket / frame_length_range_->second; |
| 917 | constraints.max += kOverheadPerPacket / frame_length_range_->first; |
| 918 | } |
Daniel Lee | 9356252 | 2019-05-03 12:40:13 | [diff] [blame] | 919 | } |
| 920 | return constraints; |
| 921 | } |
| 922 | |
ossu | 3b9ff38 | 2017-04-27 15:03:42 | [diff] [blame] | 923 | void AudioSendStream::RegisterCngPayloadType(int payload_type, |
| 924 | int clockrate_hz) { |
Niels Möller | ee5ccbc | 2019-03-06 15:47:29 | [diff] [blame] | 925 | channel_send_->RegisterCngPayloadType(payload_type, clockrate_hz); |
ossu | 3b9ff38 | 2017-04-27 15:03:42 | [diff] [blame] | 926 | } |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 927 | |
| 928 | void AudioSendStream::UpdateCachedTargetAudioBitrateConstraints() { |
| 929 | absl::optional<AudioSendStream::TargetAudioBitrateConstraints> |
| 930 | new_constraints = GetMinMaxBitrateConstraints(); |
| 931 | if (!new_constraints.has_value()) { |
| 932 | return; |
| 933 | } |
Markus Handell | 3907e7b | 2021-06-01 07:07:20 | [diff] [blame] | 934 | rtp_transport_queue_->PostTask([this, new_constraints]() { |
| 935 | RTC_DCHECK_RUN_ON(rtp_transport_queue_); |
Artem Titov | a208861 | 2021-02-03 12:33:28 | [diff] [blame] | 936 | cached_constraints_ = new_constraints; |
| 937 | }); |
| 938 | } |
| 939 | |
solenberg | c7a8b08 | 2015-10-16 21:35:07 | [diff] [blame] | 940 | } // namespace internal |
| 941 | } // namespace webrtc |