blob: d3dde5ad9c81730349e10c178b4ef91456e065df [file] [log] [blame]
pbos@webrtc.org29d58392013-05-16 12:08:031/*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
pbos@webrtc.org16e03b72013-10-28 16:32:0111#include "webrtc/video/video_send_stream.h"
pbos@webrtc.org29d58392013-05-16 12:08:0312
pbos@webrtc.orgdde16f12014-08-05 23:35:4313#include <algorithm>
pbos@webrtc.org1e92b0a2014-05-15 09:35:0614#include <sstream>
henrik.lundin@webrtc.orgba975e22013-10-23 11:04:5715#include <string>
pbos@webrtc.org29d58392013-05-16 12:08:0316#include <vector>
17
pbos@webrtc.org2b4ce3a2015-03-23 13:12:2418#include "webrtc/base/checks.h"
Peter Boström415d2cd2015-10-26 10:35:1719#include "webrtc/base/logging.h"
tommie4f96502015-10-21 06:00:4820#include "webrtc/base/trace_event.h"
mflodman0c478b32015-10-21 13:52:1621#include "webrtc/call/congestion_controller.h"
pbos@webrtc.org29d58392013-05-16 12:08:0322#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
mflodman0e7e2592015-11-13 05:02:4223#include "webrtc/modules/bitrate_controller/include/bitrate_controller.h"
Henrik Kjellander0b9e29c2015-11-16 10:12:2424#include "webrtc/modules/pacing/packet_router.h"
Peter Boströme4499152016-02-05 10:13:2825#include "webrtc/modules/utility/include/process_thread.h"
Peter Boström7623ce42015-12-09 11:13:3026#include "webrtc/video/call_stats.h"
Peter Boström4b91bd02015-06-26 04:58:1627#include "webrtc/video/video_capture_input.h"
Stefan Holmer58c664c2016-02-08 13:31:3028#include "webrtc/video/vie_remb.h"
pbos@webrtc.org16e03b72013-10-28 16:32:0129#include "webrtc/video_send_stream.h"
pbos@webrtc.org29d58392013-05-16 12:08:0330
31namespace webrtc {
mflodman949c2f02015-10-16 09:31:1132
mflodman949c2f02015-10-16 09:31:1133class RtcpIntraFrameObserver;
34class TransportFeedbackObserver;
35
pbos@webrtc.org1e92b0a2014-05-15 09:35:0636std::string
pbos@webrtc.org024e4d52014-05-15 10:03:2437VideoSendStream::Config::EncoderSettings::ToString() const {
pbos@webrtc.org1e92b0a2014-05-15 09:35:0638 std::stringstream ss;
39 ss << "{payload_name: " << payload_name;
40 ss << ", payload_type: " << payload_type;
pbos@webrtc.org2b4ce3a2015-03-23 13:12:2441 ss << ", encoder: " << (encoder != nullptr ? "(VideoEncoder)" : "nullptr");
pbos@webrtc.org1e92b0a2014-05-15 09:35:0642 ss << '}';
43 return ss.str();
44}
45
pbos@webrtc.org024e4d52014-05-15 10:03:2446std::string VideoSendStream::Config::Rtp::Rtx::ToString()
pbos@webrtc.org1e92b0a2014-05-15 09:35:0647 const {
48 std::stringstream ss;
pbos@webrtc.org32e85282015-01-15 10:09:3949 ss << "{ssrcs: [";
pbos@webrtc.org1e92b0a2014-05-15 09:35:0650 for (size_t i = 0; i < ssrcs.size(); ++i) {
51 ss << ssrcs[i];
52 if (i != ssrcs.size() - 1)
pbos@webrtc.org32e85282015-01-15 10:09:3953 ss << ", ";
pbos@webrtc.org1e92b0a2014-05-15 09:35:0654 }
pbos@webrtc.org32e85282015-01-15 10:09:3955 ss << ']';
andrew@webrtc.org8f27fcc2015-01-09 20:22:4656
pbos@webrtc.org1e92b0a2014-05-15 09:35:0657 ss << ", payload_type: " << payload_type;
58 ss << '}';
59 return ss.str();
60}
61
pbos@webrtc.org024e4d52014-05-15 10:03:2462std::string VideoSendStream::Config::Rtp::ToString() const {
pbos@webrtc.org1e92b0a2014-05-15 09:35:0663 std::stringstream ss;
pbos@webrtc.org32e85282015-01-15 10:09:3964 ss << "{ssrcs: [";
pbos@webrtc.org1e92b0a2014-05-15 09:35:0665 for (size_t i = 0; i < ssrcs.size(); ++i) {
66 ss << ssrcs[i];
67 if (i != ssrcs.size() - 1)
pbos@webrtc.org32e85282015-01-15 10:09:3968 ss << ", ";
pbos@webrtc.org1e92b0a2014-05-15 09:35:0669 }
pbos@webrtc.org32e85282015-01-15 10:09:3970 ss << ']';
pbos@webrtc.org1e92b0a2014-05-15 09:35:0671 ss << ", max_packet_size: " << max_packet_size;
pbos@webrtc.org32e85282015-01-15 10:09:3972 ss << ", extensions: [";
pbos@webrtc.org1e92b0a2014-05-15 09:35:0673 for (size_t i = 0; i < extensions.size(); ++i) {
74 ss << extensions[i].ToString();
75 if (i != extensions.size() - 1)
pbos@webrtc.org32e85282015-01-15 10:09:3976 ss << ", ";
pbos@webrtc.org1e92b0a2014-05-15 09:35:0677 }
pbos@webrtc.org32e85282015-01-15 10:09:3978 ss << ']';
pbos@webrtc.org1e92b0a2014-05-15 09:35:0679
pbos@webrtc.org32e85282015-01-15 10:09:3980 ss << ", nack: {rtp_history_ms: " << nack.rtp_history_ms << '}';
81 ss << ", fec: " << fec.ToString();
82 ss << ", rtx: " << rtx.ToString();
83 ss << ", c_name: " << c_name;
pbos@webrtc.org1e92b0a2014-05-15 09:35:0684 ss << '}';
85 return ss.str();
86}
87
pbos@webrtc.org024e4d52014-05-15 10:03:2488std::string VideoSendStream::Config::ToString() const {
pbos@webrtc.org1e92b0a2014-05-15 09:35:0689 std::stringstream ss;
90 ss << "{encoder_settings: " << encoder_settings.ToString();
91 ss << ", rtp: " << rtp.ToString();
pbos@webrtc.org32e85282015-01-15 10:09:3992 ss << ", pre_encode_callback: "
pbos@webrtc.org2b4ce3a2015-03-23 13:12:2493 << (pre_encode_callback != nullptr ? "(I420FrameCallback)" : "nullptr");
94 ss << ", post_encode_callback: " << (post_encode_callback != nullptr
95 ? "(EncodedFrameObserver)"
96 : "nullptr");
solenberg566ef242015-11-06 23:34:4997 ss << ", local_renderer: " << (local_renderer != nullptr ? "(VideoRenderer)"
pbos@webrtc.org2b4ce3a2015-03-23 13:12:2498 : "nullptr");
pbos@webrtc.org32e85282015-01-15 10:09:3999 ss << ", render_delay_ms: " << render_delay_ms;
100 ss << ", target_delay_ms: " << target_delay_ms;
101 ss << ", suspend_below_min_bitrate: " << (suspend_below_min_bitrate ? "on"
102 : "off");
pbos@webrtc.org1e92b0a2014-05-15 09:35:06103 ss << '}';
104 return ss.str();
105}
pbos@webrtc.org29d58392013-05-16 12:08:03106
Peter Boströme4499152016-02-05 10:13:28107namespace {
108
Peter Boström39593972016-02-15 10:27:15109VideoCodecType PayloadNameToCodecType(const std::string& payload_name) {
110 if (payload_name == "VP8")
111 return kVideoCodecVP8;
112 if (payload_name == "VP9")
113 return kVideoCodecVP9;
114 if (payload_name == "H264")
115 return kVideoCodecH264;
116 return kVideoCodecGeneric;
117}
118
119bool PayloadTypeSupportsSkippingFecPackets(const std::string& payload_name) {
120 switch (PayloadNameToCodecType(payload_name)) {
121 case kVideoCodecVP8:
122 case kVideoCodecVP9:
123 return true;
124 case kVideoCodecH264:
125 case kVideoCodecGeneric:
126 return false;
127 case kVideoCodecI420:
128 case kVideoCodecRED:
129 case kVideoCodecULPFEC:
130 case kVideoCodecUnknown:
131 RTC_NOTREACHED();
132 return false;
133 }
134 RTC_NOTREACHED();
135 return false;
136}
137
Peter Boströme4499152016-02-05 10:13:28138CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) {
139 CpuOveruseOptions options;
140 if (full_overuse_time) {
141 options.low_encode_usage_threshold_percent = 100;
142 options.high_encode_usage_threshold_percent = 120;
143 }
144 return options;
145}
146} // namespace
147
pbos@webrtc.org024e4d52014-05-15 10:03:24148namespace internal {
pbos@webrtc.org2bb1bda2014-07-07 13:06:48149VideoSendStream::VideoSendStream(
Peter Boström45553ae2015-05-08 11:54:38150 int num_cpu_cores,
Peter Boströmf16fcbe2015-04-30 10:16:05151 ProcessThread* module_process_thread,
mflodmane3787022015-10-21 11:24:28152 CallStats* call_stats,
mflodman0c478b32015-10-21 13:52:16153 CongestionController* congestion_controller,
Stefan Holmer58c664c2016-02-08 13:31:30154 VieRemb* remb,
mflodman0e7e2592015-11-13 05:02:42155 BitrateAllocator* bitrate_allocator,
pbos@webrtc.org2bb1bda2014-07-07 13:06:48156 const VideoSendStream::Config& config,
pbos@webrtc.orgbbe0a852014-09-19 12:30:25157 const VideoEncoderConfig& encoder_config,
Peter Boström45553ae2015-05-08 11:54:38158 const std::map<uint32_t, RtpState>& suspended_ssrcs)
sprangb4a1ae52015-12-03 16:10:08159 : stats_proxy_(Clock::GetRealTimeClock(),
160 config,
161 encoder_config.content_type),
sprang@webrtc.org40709352013-11-26 11:41:59162 encoded_frame_proxy_(config.post_encode_callback),
pbos@webrtc.org64887612013-11-14 08:58:14163 config_(config),
pbos@webrtc.org2bb1bda2014-07-07 13:06:48164 suspended_ssrcs_(suspended_ssrcs),
Peter Boströmf16fcbe2015-04-30 10:16:05165 module_process_thread_(module_process_thread),
mflodmane3787022015-10-21 11:24:28166 call_stats_(call_stats),
mflodman0c478b32015-10-21 13:52:16167 congestion_controller_(congestion_controller),
Stefan Holmer58c664c2016-02-08 13:31:30168 remb_(remb),
Peter Boströme4499152016-02-05 10:13:28169 overuse_detector_(
170 Clock::GetRealTimeClock(),
171 GetCpuOveruseOptions(config.encoder_settings.full_overuse_time),
172 this,
173 config.post_encode_callback,
174 &stats_proxy_),
Peter Boström8c66a002016-02-11 12:51:10175 vie_channel_(config.send_transport,
176 module_process_thread_,
177 &payload_router_,
178 nullptr,
Peter Boström45c44f02016-02-19 16:36:01179 &encoder_feedback_,
Peter Boström8c66a002016-02-11 12:51:10180 congestion_controller_->GetBitrateController()
181 ->CreateRtcpBandwidthObserver(),
182 congestion_controller_->GetTransportFeedbackObserver(),
183 nullptr,
184 call_stats_->rtcp_rtt_stats(),
185 congestion_controller_->pacer(),
186 congestion_controller_->packet_router(),
187 config_.rtp.ssrcs.size(),
188 true),
Peter Boström1f7d77f2016-02-11 14:30:14189 vie_receiver_(vie_channel_.vie_receiver()),
Peter Boström579e8322016-02-12 15:30:04190 vie_encoder_(num_cpu_cores,
Peter Boström0013dcc2016-02-19 19:42:19191 config_.rtp.ssrcs,
Peter Boström579e8322016-02-12 15:30:04192 module_process_thread_,
193 &stats_proxy_,
194 config.pre_encode_callback,
195 &overuse_detector_,
196 congestion_controller_->pacer(),
197 &payload_router_,
198 bitrate_allocator),
199 vcm_(vie_encoder_.vcm()),
Peter Boström723ead82016-02-22 14:14:01200 rtp_rtcp_modules_(vie_channel_.rtp_rtcp()),
Peter Boström8c66a002016-02-11 12:51:10201 input_(&vie_encoder_,
202 config_.local_renderer,
203 &stats_proxy_,
204 &overuse_detector_) {
pbosa2f30de2015-10-15 12:22:13205 LOG(LS_INFO) << "VideoSendStream: " << config_.ToString();
Stefan Holmer58c664c2016-02-08 13:31:30206
henrikg91d6ede2015-09-17 07:24:34207 RTC_DCHECK(!config_.rtp.ssrcs.empty());
Stefan Holmer58c664c2016-02-08 13:31:30208 RTC_DCHECK(module_process_thread_);
209 RTC_DCHECK(call_stats_);
210 RTC_DCHECK(congestion_controller_);
211 RTC_DCHECK(remb_);
mflodman949c2f02015-10-16 09:31:11212
Peter Boström8c66a002016-02-11 12:51:10213 RTC_CHECK(vie_encoder_.Init());
Peter Boström45c44f02016-02-19 16:36:01214 encoder_feedback_.Init(config_.rtp.ssrcs, &vie_encoder_);
Peter Boström8c66a002016-02-11 12:51:10215 RTC_CHECK(vie_channel_.Init() == 0);
mflodman949c2f02015-10-16 09:31:11216
Peter Boström8c66a002016-02-11 12:51:10217 vcm_->RegisterProtectionCallback(vie_channel_.vcm_protection_callback());
mflodman949c2f02015-10-16 09:31:11218
Peter Boström8c66a002016-02-11 12:51:10219 call_stats_->RegisterStatsObserver(vie_channel_.GetStatsObserver());
mflodman949c2f02015-10-16 09:31:11220
pbos@webrtc.org29023282013-09-11 10:14:56221 for (size_t i = 0; i < config_.rtp.extensions.size(); ++i) {
222 const std::string& extension = config_.rtp.extensions[i].name;
223 int id = config_.rtp.extensions[i].id;
Peter Boström23914fe2015-03-31 13:08:04224 // One-byte-extension local identifiers are in the range 1-14 inclusive.
henrikg91d6ede2015-09-17 07:24:34225 RTC_DCHECK_GE(id, 1);
226 RTC_DCHECK_LE(id, 14);
pbos@webrtc.orgce90eff2013-11-20 11:48:56227 if (extension == RtpExtension::kTOffset) {
Peter Boström8c66a002016-02-11 12:51:10228 RTC_CHECK_EQ(0, vie_channel_.SetSendTimestampOffsetStatus(true, id));
pbos@webrtc.orgce90eff2013-11-20 11:48:56229 } else if (extension == RtpExtension::kAbsSendTime) {
Peter Boström8c66a002016-02-11 12:51:10230 RTC_CHECK_EQ(0, vie_channel_.SetSendAbsoluteSendTimeStatus(true, id));
guoweis@webrtc.orgfdd10572015-03-12 20:50:57231 } else if (extension == RtpExtension::kVideoRotation) {
Peter Boström8c66a002016-02-11 12:51:10232 RTC_CHECK_EQ(0, vie_channel_.SetSendVideoRotationStatus(true, id));
sprang867fb522015-08-03 11:38:41233 } else if (extension == RtpExtension::kTransportSequenceNumber) {
Peter Boström8c66a002016-02-11 12:51:10234 RTC_CHECK_EQ(0, vie_channel_.SetSendTransportSequenceNumber(true, id));
pbos@webrtc.org29023282013-09-11 10:14:56235 } else {
pbos@webrtc.org2b4ce3a2015-03-23 13:12:24236 RTC_NOTREACHED() << "Registering unsupported RTP extension.";
pbos@webrtc.org29023282013-09-11 10:14:56237 }
238 }
pbos@webrtc.org29d58392013-05-16 12:08:03239
Peter Boström723ead82016-02-22 14:14:01240 remb_->AddRembSender(rtp_rtcp_modules_[0]);
241 rtp_rtcp_modules_[0]->SetREMBStatus(true);
mflodman@webrtc.org92c27932013-12-13 16:36:28242
pbos@webrtc.org0e63e762013-09-20 11:56:26243 // Enable NACK, FEC or both.
pbosba8c15b2015-07-14 16:36:34244 const bool enable_protection_nack = config_.rtp.nack.rtp_history_ms > 0;
Peter Boström39593972016-02-15 10:27:15245 bool enable_protection_fec = config_.rtp.fec.red_payload_type != -1;
246 // Payload types without picture ID cannot determine that a stream is complete
247 // without retransmitting FEC, so using FEC + NACK for H.264 (for instance) is
248 // a waste of bandwidth since FEC packets still have to be transmitted. Note
249 // that this is not the case with FLEXFEC.
250 if (enable_protection_nack &&
251 !PayloadTypeSupportsSkippingFecPackets(
252 config_.encoder_settings.payload_name)) {
253 LOG(LS_WARNING) << "Transmitting payload type without picture ID using"
254 "NACK+FEC is a waste of bandwidth since FEC packets "
255 "also have to be retransmitted. Disabling FEC.";
256 enable_protection_fec = false;
257 }
pbosba8c15b2015-07-14 16:36:34258 // TODO(changbin): Should set RTX for RED mapping in RTP sender in future.
Peter Boström8c66a002016-02-11 12:51:10259 vie_channel_.SetProtectionMode(enable_protection_nack, enable_protection_fec,
Peter Boström723ead82016-02-22 14:14:01260 config_.rtp.fec.red_payload_type,
261 config_.rtp.fec.ulpfec_payload_type);
Peter Boström8c66a002016-02-11 12:51:10262 vie_encoder_.SetProtectionMethod(enable_protection_nack,
Peter Boström723ead82016-02-22 14:14:01263 enable_protection_fec);
pbos@webrtc.org0e63e762013-09-20 11:56:26264
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09265 ConfigureSsrcs();
266
Peter Boström723ead82016-02-22 14:14:01267 // TODO(pbos): Should we set CNAME on all RTP modules?
268 rtp_rtcp_modules_.front()->SetCNAME(config_.rtp.c_name.c_str());
sprang@webrtc.org25fce9a2013-10-16 13:29:14269 // 28 to match packet overhead in ModuleRtpRtcpImpl.
Peter Boström723ead82016-02-22 14:14:01270 static const size_t kRtpPacketSizeOverhead = 28;
271 RTC_DCHECK_LE(config_.rtp.max_packet_size, 0xFFFFu + kRtpPacketSizeOverhead);
272 const uint16_t mtu = static_cast<uint16_t>(config_.rtp.max_packet_size +
273 kRtpPacketSizeOverhead);
274 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
275 rtp_rtcp->RegisterRtcpStatisticsCallback(&stats_proxy_);
276 rtp_rtcp->RegisterSendChannelRtpStatisticsCallback(&stats_proxy_);
277 rtp_rtcp->SetMaxTransferUnit(mtu);
278 }
pbos@webrtc.org29d58392013-05-16 12:08:03279
henrikg91d6ede2015-09-17 07:24:34280 RTC_DCHECK(config.encoder_settings.encoder != nullptr);
281 RTC_DCHECK_GE(config.encoder_settings.payload_type, 0);
282 RTC_DCHECK_LE(config.encoder_settings.payload_type, 127);
Peter Boström8c66a002016-02-11 12:51:10283 RTC_CHECK_EQ(0, vie_encoder_.RegisterExternalEncoder(
henrikg91d6ede2015-09-17 07:24:34284 config.encoder_settings.encoder,
285 config.encoder_settings.payload_type,
286 config.encoder_settings.internal_source));
stefan@webrtc.org360e3762013-08-22 09:29:56287
henrikg91d6ede2015-09-17 07:24:34288 RTC_CHECK(ReconfigureVideoEncoder(encoder_config));
pbos@webrtc.orgfe1ef932013-10-21 10:34:43289
Peter Boström8c66a002016-02-11 12:51:10290 vie_channel_.RegisterSendSideDelayObserver(&stats_proxy_);
stefan@webrtc.org168f23f2014-07-11 13:44:02291
Peter Boström94cc1fe2015-04-29 12:08:41292 if (config_.post_encode_callback)
Peter Boström8c66a002016-02-11 12:51:10293 vie_encoder_.RegisterPostEncodeImageCallback(&encoded_frame_proxy_);
henrik.lundin@webrtc.orgba975e22013-10-23 11:04:57294
Peter Boströme23e7372015-10-08 09:44:14295 if (config_.suspend_below_min_bitrate)
Peter Boström8c66a002016-02-11 12:51:10296 vie_encoder_.SuspendBelowMinBitrate();
sprang@webrtc.orgccd42842014-01-07 09:54:34297
Peter Boström8c66a002016-02-11 12:51:10298 vie_channel_.RegisterRtcpPacketTypeCounterObserver(&stats_proxy_);
299 vie_channel_.RegisterSendBitrateObserver(&stats_proxy_);
300 vie_channel_.RegisterSendFrameCountObserver(&stats_proxy_);
Peter Boströme4499152016-02-05 10:13:28301
302 module_process_thread_->RegisterModule(&overuse_detector_);
pbos@webrtc.org29d58392013-05-16 12:08:03303}
304
305VideoSendStream::~VideoSendStream() {
pbosa2f30de2015-10-15 12:22:13306 LOG(LS_INFO) << "~VideoSendStream: " << config_.ToString();
Peter Boströmca835252016-02-11 14:59:46307 Stop();
308
Peter Boströme4499152016-02-05 10:13:28309 module_process_thread_->DeRegisterModule(&overuse_detector_);
Peter Boström8c66a002016-02-11 12:51:10310 vie_channel_.RegisterSendFrameCountObserver(nullptr);
311 vie_channel_.RegisterSendBitrateObserver(nullptr);
312 vie_channel_.RegisterRtcpPacketTypeCounterObserver(nullptr);
sprang@webrtc.orgccd42842014-01-07 09:54:34313
Peter Boström723ead82016-02-22 14:14:01314 vie_encoder_.DeRegisterExternalEncoder(config_.encoder_settings.payload_type);
stefan@webrtc.org360e3762013-08-22 09:29:56315
Peter Boström8c66a002016-02-11 12:51:10316 call_stats_->DeregisterStatsObserver(vie_channel_.GetStatsObserver());
Peter Boström723ead82016-02-22 14:14:01317 rtp_rtcp_modules_[0]->SetREMBStatus(false);
318 remb_->RemoveRembSender(rtp_rtcp_modules_[0]);
mflodman949c2f02015-10-16 09:31:11319
Peter Boström45c44f02016-02-19 16:36:01320 // ViEChannel outlives ViEEncoder so remove encoder from feedback before
321 // destruction.
322 encoder_feedback_.TearDown();
mflodman949c2f02015-10-16 09:31:11323
mflodman0c478b32015-10-21 13:52:16324 congestion_controller_->GetRemoteBitrateEstimator(false)->RemoveStream(
Peter Boström1f7d77f2016-02-11 14:30:14325 vie_receiver_->GetRemoteSsrc());
pbos@webrtc.org29d58392013-05-16 12:08:03326}
327
Peter Boström4b91bd02015-06-26 04:58:16328VideoCaptureInput* VideoSendStream::Input() {
Peter Boström8c66a002016-02-11 12:51:10329 return &input_;
pbos@webrtc.org29d58392013-05-16 12:08:03330}
331
pbos@webrtc.orga5c8d2c2014-04-24 11:13:21332void VideoSendStream::Start() {
Peter Boström8c66a002016-02-11 12:51:10333 vie_encoder_.Pause();
334 if (vie_channel_.StartSend() == 0) {
Peter Boström45553ae2015-05-08 11:54:38335 // Was not already started, trigger a keyframe.
Peter Boström8c66a002016-02-11 12:51:10336 vie_encoder_.SendKeyFrame();
Peter Boström45553ae2015-05-08 11:54:38337 }
Peter Boström8c66a002016-02-11 12:51:10338 vie_encoder_.Restart();
Peter Boströmca835252016-02-11 14:59:46339 vie_receiver_->StartReceive();
pbos@webrtc.org29d58392013-05-16 12:08:03340}
341
pbos@webrtc.orga5c8d2c2014-04-24 11:13:21342void VideoSendStream::Stop() {
Peter Boström45553ae2015-05-08 11:54:38343 // TODO(pbos): Make sure the encoder stops here.
Peter Boström8c66a002016-02-11 12:51:10344 vie_channel_.StopSend();
Peter Boströmca835252016-02-11 14:59:46345 vie_receiver_->StopReceive();
pbos@webrtc.org29d58392013-05-16 12:08:03346}
347
pbos@webrtc.orgf577ae92014-03-19 08:43:57348bool VideoSendStream::ReconfigureVideoEncoder(
pbos@webrtc.orgbbe0a852014-09-19 12:30:25349 const VideoEncoderConfig& config) {
pbos@webrtc.org50fe3592015-01-29 12:33:07350 TRACE_EVENT0("webrtc", "VideoSendStream::(Re)configureVideoEncoder");
pbos@webrtc.orgad3b5a52014-10-24 09:23:21351 LOG(LS_INFO) << "(Re)configureVideoEncoder: " << config.ToString();
pbos@webrtc.orgbbe0a852014-09-19 12:30:25352 const std::vector<VideoStream>& streams = config.streams;
henrikg91d6ede2015-09-17 07:24:34353 RTC_DCHECK(!streams.empty());
354 RTC_DCHECK_GE(config_.rtp.ssrcs.size(), streams.size());
pbos@webrtc.org64887612013-11-14 08:58:14355
pbos@webrtc.orgf577ae92014-03-19 08:43:57356 VideoCodec video_codec;
357 memset(&video_codec, 0, sizeof(video_codec));
Peter Boström39593972016-02-15 10:27:15358 video_codec.codecType =
359 PayloadNameToCodecType(config_.encoder_settings.payload_name);
pbos@webrtc.orgb7ed7792014-10-31 13:08:10360
pbos@webrtc.orgbbe0a852014-09-19 12:30:25361 switch (config.content_type) {
Erik Språng143cec12015-04-28 08:01:41362 case VideoEncoderConfig::ContentType::kRealtimeVideo:
pbos@webrtc.orgbbe0a852014-09-19 12:30:25363 video_codec.mode = kRealtimeVideo;
364 break;
Erik Språng143cec12015-04-28 08:01:41365 case VideoEncoderConfig::ContentType::kScreen:
pbos@webrtc.orgbbe0a852014-09-19 12:30:25366 video_codec.mode = kScreensharing;
pbos@webrtc.orgb7ed7792014-10-31 13:08:10367 if (config.streams.size() == 1 &&
368 config.streams[0].temporal_layer_thresholds_bps.size() == 1) {
369 video_codec.targetBitrate =
370 config.streams[0].temporal_layer_thresholds_bps[0] / 1000;
371 }
pbos@webrtc.orgbbe0a852014-09-19 12:30:25372 break;
373 }
pbos@webrtc.orgf577ae92014-03-19 08:43:57374
375 if (video_codec.codecType == kVideoCodecVP8) {
pbos@webrtc.org6cd6ba82014-09-18 12:42:28376 video_codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings();
marpan@webrtc.org5b883172014-11-01 06:10:48377 } else if (video_codec.codecType == kVideoCodecVP9) {
378 video_codec.codecSpecific.VP9 = VideoEncoder::GetDefaultVp9Settings();
stefan@webrtc.org79c33592014-08-06 09:24:53379 } else if (video_codec.codecType == kVideoCodecH264) {
pbos@webrtc.org6cd6ba82014-09-18 12:42:28380 video_codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings();
pbos@webrtc.orgf577ae92014-03-19 08:43:57381 }
382
pbos@webrtc.org91f17522014-07-10 10:13:37383 if (video_codec.codecType == kVideoCodecVP8) {
pbos@webrtc.org2b4ce3a2015-03-23 13:12:24384 if (config.encoder_specific_settings != nullptr) {
pbos@webrtc.orgbbe0a852014-09-19 12:30:25385 video_codec.codecSpecific.VP8 = *reinterpret_cast<const VideoCodecVP8*>(
386 config.encoder_specific_settings);
pbos@webrtc.org91f17522014-07-10 10:13:37387 }
pbos@webrtc.org759982d2014-09-22 09:32:46388 video_codec.codecSpecific.VP8.numberOfTemporalLayers =
pbos@webrtc.orgb7ed7792014-10-31 13:08:10389 static_cast<unsigned char>(
390 streams.back().temporal_layer_thresholds_bps.size() + 1);
pbos@webrtc.orgb9557a92015-03-20 19:52:56391 } else if (video_codec.codecType == kVideoCodecVP9) {
pbos@webrtc.org2b4ce3a2015-03-23 13:12:24392 if (config.encoder_specific_settings != nullptr) {
pbos@webrtc.orgb9557a92015-03-20 19:52:56393 video_codec.codecSpecific.VP9 = *reinterpret_cast<const VideoCodecVP9*>(
394 config.encoder_specific_settings);
philipelcfc319b2015-11-10 15:17:23395 if (video_codec.mode == kScreensharing) {
396 video_codec.codecSpecific.VP9.flexibleMode = true;
397 // For now VP9 screensharing use 1 temporal and 2 spatial layers.
398 RTC_DCHECK_EQ(video_codec.codecSpecific.VP9.numberOfTemporalLayers, 1);
399 RTC_DCHECK_EQ(video_codec.codecSpecific.VP9.numberOfSpatialLayers, 2);
400 }
pbos@webrtc.orgb9557a92015-03-20 19:52:56401 }
402 video_codec.codecSpecific.VP9.numberOfTemporalLayers =
403 static_cast<unsigned char>(
404 streams.back().temporal_layer_thresholds_bps.size() + 1);
405 } else if (video_codec.codecType == kVideoCodecH264) {
pbos@webrtc.org2b4ce3a2015-03-23 13:12:24406 if (config.encoder_specific_settings != nullptr) {
pbos@webrtc.orgb9557a92015-03-20 19:52:56407 video_codec.codecSpecific.H264 = *reinterpret_cast<const VideoCodecH264*>(
408 config.encoder_specific_settings);
409 }
pbos@webrtc.org91f17522014-07-10 10:13:37410 } else {
411 // TODO(pbos): Support encoder_settings codec-agnostically.
henrikg91d6ede2015-09-17 07:24:34412 RTC_DCHECK(config.encoder_specific_settings == nullptr)
pbos@webrtc.org2b4ce3a2015-03-23 13:12:24413 << "Encoder-specific settings for codec type not wired up.";
pbos@webrtc.org91f17522014-07-10 10:13:37414 }
415
pbos@webrtc.orgf577ae92014-03-19 08:43:57416 strncpy(video_codec.plName,
417 config_.encoder_settings.payload_name.c_str(),
418 kPayloadNameSize - 1);
419 video_codec.plName[kPayloadNameSize - 1] = '\0';
420 video_codec.plType = config_.encoder_settings.payload_type;
421 video_codec.numberOfSimulcastStreams =
422 static_cast<unsigned char>(streams.size());
423 video_codec.minBitrate = streams[0].min_bitrate_bps / 1000;
henrikg91d6ede2015-09-17 07:24:34424 RTC_DCHECK_LE(streams.size(), static_cast<size_t>(kMaxSimulcastStreams));
sprangce4aef12015-11-02 15:23:20425 if (video_codec.codecType == kVideoCodecVP9) {
426 // If the vector is empty, bitrates will be configured automatically.
427 RTC_DCHECK(config.spatial_layers.empty() ||
428 config.spatial_layers.size() ==
429 video_codec.codecSpecific.VP9.numberOfSpatialLayers);
430 RTC_DCHECK_LE(video_codec.codecSpecific.VP9.numberOfSpatialLayers,
431 kMaxSimulcastStreams);
432 for (size_t i = 0; i < config.spatial_layers.size(); ++i)
433 video_codec.spatialLayers[i] = config.spatial_layers[i];
434 }
pbos@webrtc.orgf577ae92014-03-19 08:43:57435 for (size_t i = 0; i < streams.size(); ++i) {
436 SimulcastStream* sim_stream = &video_codec.simulcastStream[i];
henrikg91d6ede2015-09-17 07:24:34437 RTC_DCHECK_GT(streams[i].width, 0u);
438 RTC_DCHECK_GT(streams[i].height, 0u);
439 RTC_DCHECK_GT(streams[i].max_framerate, 0);
pbos@webrtc.orgf577ae92014-03-19 08:43:57440 // Different framerates not supported per stream at the moment.
henrikg91d6ede2015-09-17 07:24:34441 RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate);
442 RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0);
443 RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps);
444 RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps);
445 RTC_DCHECK_GE(streams[i].max_qp, 0);
pbos@webrtc.orgf577ae92014-03-19 08:43:57446
mflodmand1590b22015-12-09 15:07:59447 sim_stream->width = static_cast<uint16_t>(streams[i].width);
448 sim_stream->height = static_cast<uint16_t>(streams[i].height);
pbos@webrtc.orgf577ae92014-03-19 08:43:57449 sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000;
450 sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000;
451 sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000;
452 sim_stream->qpMax = streams[i].max_qp;
pbos@webrtc.orgb7ed7792014-10-31 13:08:10453 sim_stream->numberOfTemporalLayers = static_cast<unsigned char>(
454 streams[i].temporal_layer_thresholds_bps.size() + 1);
pbos@webrtc.orgf577ae92014-03-19 08:43:57455
456 video_codec.width = std::max(video_codec.width,
mflodmand1590b22015-12-09 15:07:59457 static_cast<uint16_t>(streams[i].width));
pbos@webrtc.orgf577ae92014-03-19 08:43:57458 video_codec.height = std::max(
mflodmand1590b22015-12-09 15:07:59459 video_codec.height, static_cast<uint16_t>(streams[i].height));
pbos@webrtc.orgf577ae92014-03-19 08:43:57460 video_codec.minBitrate =
mflodmand1590b22015-12-09 15:07:59461 std::min(static_cast<uint16_t>(video_codec.minBitrate),
462 static_cast<uint16_t>(streams[i].min_bitrate_bps / 1000));
pbos@webrtc.orgf577ae92014-03-19 08:43:57463 video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000;
464 video_codec.qpMax = std::max(video_codec.qpMax,
465 static_cast<unsigned int>(streams[i].max_qp));
466 }
Stefan Holmere5904162015-03-26 10:11:06467
468 // Set to zero to not update the bitrate controller from ViEEncoder, as
469 // the bitrate controller is already set from Call.
470 video_codec.startBitrate = 0;
pbos@webrtc.orgf577ae92014-03-19 08:43:57471
henrikg91d6ede2015-09-17 07:24:34472 RTC_DCHECK_GT(streams[0].max_framerate, 0);
pbos@webrtc.org6ae48c62014-06-06 10:49:19473 video_codec.maxFramerate = streams[0].max_framerate;
pbos@webrtc.orgf577ae92014-03-19 08:43:57474
Peter Boström5cb9ce42015-05-05 13:16:30475 if (!SetSendCodec(video_codec))
pbos@webrtc.org32452b22014-10-22 12:15:24476 return false;
477
Peter Boström20f3f942015-05-15 09:33:39478 // Clear stats for disabled layers.
479 for (size_t i = video_codec.numberOfSimulcastStreams;
480 i < config_.rtp.ssrcs.size(); ++i) {
481 stats_proxy_.OnInactiveSsrc(config_.rtp.ssrcs[i]);
482 }
483
sprangb4a1ae52015-12-03 16:10:08484 stats_proxy_.SetContentType(config.content_type);
485
henrikg91d6ede2015-09-17 07:24:34486 RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0);
Peter Boström8c66a002016-02-11 12:51:10487 vie_encoder_.SetMinTransmitBitrate(config.min_transmit_bitrate_bps / 1000);
pbos@webrtc.orgad3b5a52014-10-24 09:23:21488
pbos@webrtc.org00873182014-11-25 14:03:34489 encoder_config_ = config;
pbos@webrtc.org32452b22014-10-22 12:15:24490 return true;
pbos@webrtc.org29d58392013-05-16 12:08:03491}
492
pbos@webrtc.orgbbb07e62013-08-05 12:01:36493bool VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
Peter Boström1f7d77f2016-02-11 14:30:14494 return vie_receiver_->DeliverRtcp(packet, length);
pbos@webrtc.orgbbb07e62013-08-05 12:01:36495}
sprang@webrtc.orgccd42842014-01-07 09:54:34496
pbos@webrtc.org273a4142014-12-01 15:23:21497VideoSendStream::Stats VideoSendStream::GetStats() {
stefan@webrtc.org168f23f2014-07-11 13:44:02498 return stats_proxy_.GetStats();
sprang@webrtc.orgccd42842014-01-07 09:54:34499}
500
solenberge5269742015-09-08 12:13:22501void VideoSendStream::OveruseDetected() {
502 if (config_.overuse_callback)
503 config_.overuse_callback->OnLoadUpdate(LoadObserver::kOveruse);
504}
505
506void VideoSendStream::NormalUsage() {
507 if (config_.overuse_callback)
508 config_.overuse_callback->OnLoadUpdate(LoadObserver::kUnderuse);
509}
510
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09511void VideoSendStream::ConfigureSsrcs() {
Peter Boström723ead82016-02-22 14:14:01512 // Configure regular SSRCs.
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09513 for (size_t i = 0; i < config_.rtp.ssrcs.size(); ++i) {
514 uint32_t ssrc = config_.rtp.ssrcs[i];
Peter Boström723ead82016-02-22 14:14:01515 RtpRtcp* const rtp_rtcp = rtp_rtcp_modules_[i];
516 rtp_rtcp->SetSSRC(ssrc);
517
518 // Restore RTP state if previous existed.
pbos@webrtc.org2bb1bda2014-07-07 13:06:48519 RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc);
520 if (it != suspended_ssrcs_.end())
Peter Boström723ead82016-02-22 14:14:01521 rtp_rtcp->SetRtpStateForSsrc(ssrc, it->second);
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09522 }
523
Peter Boström723ead82016-02-22 14:14:01524 // Set up RTX if available.
525 if (config_.rtp.rtx.ssrcs.empty())
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09526 return;
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09527
Peter Boström723ead82016-02-22 14:14:01528 // Configure RTX SSRCs.
henrikg91d6ede2015-09-17 07:24:34529 RTC_DCHECK_EQ(config_.rtp.rtx.ssrcs.size(), config_.rtp.ssrcs.size());
pbos@webrtc.org2bb1bda2014-07-07 13:06:48530 for (size_t i = 0; i < config_.rtp.rtx.ssrcs.size(); ++i) {
531 uint32_t ssrc = config_.rtp.rtx.ssrcs[i];
Peter Boström723ead82016-02-22 14:14:01532 RtpRtcp* const rtp_rtcp = rtp_rtcp_modules_[i];
533 rtp_rtcp->SetRtxSsrc(ssrc);
pbos@webrtc.org2bb1bda2014-07-07 13:06:48534 RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc);
535 if (it != suspended_ssrcs_.end())
Peter Boström723ead82016-02-22 14:14:01536 rtp_rtcp->SetRtpStateForSsrc(ssrc, it->second);
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09537 }
538
Peter Boström723ead82016-02-22 14:14:01539 // Configure RTX payload types.
henrikg91d6ede2015-09-17 07:24:34540 RTC_DCHECK_GE(config_.rtp.rtx.payload_type, 0);
Peter Boström723ead82016-02-22 14:14:01541 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
542 rtp_rtcp->SetRtxSendPayloadType(config_.rtp.rtx.payload_type,
543 config_.encoder_settings.payload_type);
544 rtp_rtcp->SetRtxSendStatus(kRtxRetransmitted | kRtxRedundantPayloads);
545 }
Stefan Holmer10880012016-02-03 12:29:59546 if (config_.rtp.fec.red_payload_type != -1 &&
547 config_.rtp.fec.red_rtx_payload_type != -1) {
Peter Boström723ead82016-02-22 14:14:01548 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
549 rtp_rtcp->SetRtxSendPayloadType(config_.rtp.fec.red_rtx_payload_type,
550 config_.rtp.fec.red_payload_type);
551 }
Stefan Holmer10880012016-02-03 12:29:59552 }
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09553}
554
pbos@webrtc.org2bb1bda2014-07-07 13:06:48555std::map<uint32_t, RtpState> VideoSendStream::GetRtpStates() const {
556 std::map<uint32_t, RtpState> rtp_states;
557 for (size_t i = 0; i < config_.rtp.ssrcs.size(); ++i) {
558 uint32_t ssrc = config_.rtp.ssrcs[i];
Peter Boström8c66a002016-02-11 12:51:10559 rtp_states[ssrc] = vie_channel_.GetRtpStateForSsrc(ssrc);
pbos@webrtc.org2bb1bda2014-07-07 13:06:48560 }
561
562 for (size_t i = 0; i < config_.rtp.rtx.ssrcs.size(); ++i) {
563 uint32_t ssrc = config_.rtp.rtx.ssrcs[i];
Peter Boström8c66a002016-02-11 12:51:10564 rtp_states[ssrc] = vie_channel_.GetRtpStateForSsrc(ssrc);
pbos@webrtc.org2bb1bda2014-07-07 13:06:48565 }
566
567 return rtp_states;
568}
569
Jelena Marusiccd670222015-07-16 07:30:09570void VideoSendStream::SignalNetworkState(NetworkState state) {
pbos@webrtc.org26c0c412014-09-03 16:17:12571 // When network goes up, enable RTCP status before setting transmission state.
572 // When it goes down, disable RTCP afterwards. This ensures that any packets
573 // sent due to the network state changed will not be dropped.
Peter Boström723ead82016-02-22 14:14:01574 if (state == kNetworkUp) {
575 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_)
576 rtp_rtcp->SetRTCPStatus(config_.rtp.rtcp_mode);
577 }
Peter Boström8c66a002016-02-11 12:51:10578 vie_encoder_.SetNetworkTransmissionState(state == kNetworkUp);
Peter Boström723ead82016-02-22 14:14:01579 if (state == kNetworkDown) {
580 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_)
581 rtp_rtcp->SetRTCPStatus(RtcpMode::kOff);
582 }
stefan@webrtc.org0bae1fa2014-11-05 14:05:29583}
pbos@webrtc.org2b19f062014-12-11 13:26:09584
mflodman0e7e2592015-11-13 05:02:42585int VideoSendStream::GetPaddingNeededBps() const {
Peter Boström8c66a002016-02-11 12:51:10586 return vie_encoder_.GetPaddingNeededBps();
mflodman0e7e2592015-11-13 05:02:42587}
588
Peter Boström5cb9ce42015-05-05 13:16:30589bool VideoSendStream::SetSendCodec(VideoCodec video_codec) {
mflodmanc4a1c372015-11-06 12:33:51590 static const int kEncoderMinBitrate = 30;
Peter Boström5cb9ce42015-05-05 13:16:30591 if (video_codec.maxBitrate == 0) {
592 // Unset max bitrate -> cap to one bit per pixel.
593 video_codec.maxBitrate =
594 (video_codec.width * video_codec.height * video_codec.maxFramerate) /
595 1000;
596 }
597
mflodmanc4a1c372015-11-06 12:33:51598 if (video_codec.minBitrate < kEncoderMinBitrate)
599 video_codec.minBitrate = kEncoderMinBitrate;
600 if (video_codec.maxBitrate < kEncoderMinBitrate)
601 video_codec.maxBitrate = kEncoderMinBitrate;
Peter Boström5cb9ce42015-05-05 13:16:30602
603 // Stop the media flow while reconfiguring.
Peter Boström8c66a002016-02-11 12:51:10604 vie_encoder_.Pause();
Peter Boström5cb9ce42015-05-05 13:16:30605
Peter Boström8c66a002016-02-11 12:51:10606 if (vie_encoder_.SetEncoder(video_codec) != 0) {
Peter Boström5cb9ce42015-05-05 13:16:30607 LOG(LS_ERROR) << "Failed to set encoder.";
608 return false;
609 }
610
Peter Boström8c66a002016-02-11 12:51:10611 if (vie_channel_.SetSendCodec(video_codec, false) != 0) {
Peter Boström5cb9ce42015-05-05 13:16:30612 LOG(LS_ERROR) << "Failed to set send codec.";
613 return false;
614 }
615
Peter Boström5cb9ce42015-05-05 13:16:30616 // Restart the media flow
Peter Boström8c66a002016-02-11 12:51:10617 vie_encoder_.Restart();
Peter Boström5cb9ce42015-05-05 13:16:30618
619 return true;
620}
pbos@webrtc.org29d58392013-05-16 12:08:03621} // namespace internal
622} // namespace webrtc