blob: 66a6a321a14c652c48ce8e1715243509c81bbca2 [file] [log] [blame]
pbos@webrtc.org29d58392013-05-16 12:08:031/*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
pbos@webrtc.org16e03b72013-10-28 16:32:0111#include "webrtc/video/video_send_stream.h"
pbos@webrtc.org29d58392013-05-16 12:08:0312
pbos@webrtc.orgdde16f12014-08-05 23:35:4313#include <algorithm>
pbos@webrtc.org1e92b0a2014-05-15 09:35:0614#include <sstream>
henrik.lundin@webrtc.orgba975e22013-10-23 11:04:5715#include <string>
pbos@webrtc.org29d58392013-05-16 12:08:0316#include <vector>
17
pbos@webrtc.org2b4ce3a2015-03-23 13:12:2418#include "webrtc/base/checks.h"
Peter Boström415d2cd2015-10-26 10:35:1719#include "webrtc/base/logging.h"
tommie4f96502015-10-21 06:00:4820#include "webrtc/base/trace_event.h"
pbos@webrtc.org29d58392013-05-16 12:08:0321#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
mflodman0e7e2592015-11-13 05:02:4222#include "webrtc/modules/bitrate_controller/include/bitrate_controller.h"
Stefan Holmer80e12072016-02-23 12:30:4223#include "webrtc/modules/congestion_controller/include/congestion_controller.h"
Henrik Kjellander0b9e29c2015-11-16 10:12:2424#include "webrtc/modules/pacing/packet_router.h"
Peter Boström9c017252016-02-26 15:26:2025#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
Peter Boströme4499152016-02-05 10:13:2826#include "webrtc/modules/utility/include/process_thread.h"
Peter Boström7623ce42015-12-09 11:13:3027#include "webrtc/video/call_stats.h"
Peter Boström4b91bd02015-06-26 04:58:1628#include "webrtc/video/video_capture_input.h"
Stefan Holmer58c664c2016-02-08 13:31:3029#include "webrtc/video/vie_remb.h"
pbos@webrtc.org16e03b72013-10-28 16:32:0130#include "webrtc/video_send_stream.h"
pbos@webrtc.org29d58392013-05-16 12:08:0331
32namespace webrtc {
mflodman949c2f02015-10-16 09:31:1133
mflodman949c2f02015-10-16 09:31:1134class RtcpIntraFrameObserver;
35class TransportFeedbackObserver;
36
pbos@webrtc.org1e92b0a2014-05-15 09:35:0637std::string
pbos@webrtc.org024e4d52014-05-15 10:03:2438VideoSendStream::Config::EncoderSettings::ToString() const {
pbos@webrtc.org1e92b0a2014-05-15 09:35:0639 std::stringstream ss;
40 ss << "{payload_name: " << payload_name;
41 ss << ", payload_type: " << payload_type;
pbos@webrtc.org2b4ce3a2015-03-23 13:12:2442 ss << ", encoder: " << (encoder != nullptr ? "(VideoEncoder)" : "nullptr");
pbos@webrtc.org1e92b0a2014-05-15 09:35:0643 ss << '}';
44 return ss.str();
45}
46
pbos@webrtc.org024e4d52014-05-15 10:03:2447std::string VideoSendStream::Config::Rtp::Rtx::ToString()
pbos@webrtc.org1e92b0a2014-05-15 09:35:0648 const {
49 std::stringstream ss;
pbos@webrtc.org32e85282015-01-15 10:09:3950 ss << "{ssrcs: [";
pbos@webrtc.org1e92b0a2014-05-15 09:35:0651 for (size_t i = 0; i < ssrcs.size(); ++i) {
52 ss << ssrcs[i];
53 if (i != ssrcs.size() - 1)
pbos@webrtc.org32e85282015-01-15 10:09:3954 ss << ", ";
pbos@webrtc.org1e92b0a2014-05-15 09:35:0655 }
pbos@webrtc.org32e85282015-01-15 10:09:3956 ss << ']';
andrew@webrtc.org8f27fcc2015-01-09 20:22:4657
pbos@webrtc.org1e92b0a2014-05-15 09:35:0658 ss << ", payload_type: " << payload_type;
59 ss << '}';
60 return ss.str();
61}
62
pbos@webrtc.org024e4d52014-05-15 10:03:2463std::string VideoSendStream::Config::Rtp::ToString() const {
pbos@webrtc.org1e92b0a2014-05-15 09:35:0664 std::stringstream ss;
pbos@webrtc.org32e85282015-01-15 10:09:3965 ss << "{ssrcs: [";
pbos@webrtc.org1e92b0a2014-05-15 09:35:0666 for (size_t i = 0; i < ssrcs.size(); ++i) {
67 ss << ssrcs[i];
68 if (i != ssrcs.size() - 1)
pbos@webrtc.org32e85282015-01-15 10:09:3969 ss << ", ";
pbos@webrtc.org1e92b0a2014-05-15 09:35:0670 }
pbos@webrtc.org32e85282015-01-15 10:09:3971 ss << ']';
pbos@webrtc.org1e92b0a2014-05-15 09:35:0672 ss << ", max_packet_size: " << max_packet_size;
pbos@webrtc.org32e85282015-01-15 10:09:3973 ss << ", extensions: [";
pbos@webrtc.org1e92b0a2014-05-15 09:35:0674 for (size_t i = 0; i < extensions.size(); ++i) {
75 ss << extensions[i].ToString();
76 if (i != extensions.size() - 1)
pbos@webrtc.org32e85282015-01-15 10:09:3977 ss << ", ";
pbos@webrtc.org1e92b0a2014-05-15 09:35:0678 }
pbos@webrtc.org32e85282015-01-15 10:09:3979 ss << ']';
pbos@webrtc.org1e92b0a2014-05-15 09:35:0680
pbos@webrtc.org32e85282015-01-15 10:09:3981 ss << ", nack: {rtp_history_ms: " << nack.rtp_history_ms << '}';
82 ss << ", fec: " << fec.ToString();
83 ss << ", rtx: " << rtx.ToString();
84 ss << ", c_name: " << c_name;
pbos@webrtc.org1e92b0a2014-05-15 09:35:0685 ss << '}';
86 return ss.str();
87}
88
pbos@webrtc.org024e4d52014-05-15 10:03:2489std::string VideoSendStream::Config::ToString() const {
pbos@webrtc.org1e92b0a2014-05-15 09:35:0690 std::stringstream ss;
91 ss << "{encoder_settings: " << encoder_settings.ToString();
92 ss << ", rtp: " << rtp.ToString();
pbos@webrtc.org32e85282015-01-15 10:09:3993 ss << ", pre_encode_callback: "
pbos@webrtc.org2b4ce3a2015-03-23 13:12:2494 << (pre_encode_callback != nullptr ? "(I420FrameCallback)" : "nullptr");
95 ss << ", post_encode_callback: " << (post_encode_callback != nullptr
96 ? "(EncodedFrameObserver)"
97 : "nullptr");
solenberg566ef242015-11-06 23:34:4998 ss << ", local_renderer: " << (local_renderer != nullptr ? "(VideoRenderer)"
pbos@webrtc.org2b4ce3a2015-03-23 13:12:2499 : "nullptr");
pbos@webrtc.org32e85282015-01-15 10:09:39100 ss << ", render_delay_ms: " << render_delay_ms;
101 ss << ", target_delay_ms: " << target_delay_ms;
102 ss << ", suspend_below_min_bitrate: " << (suspend_below_min_bitrate ? "on"
103 : "off");
pbos@webrtc.org1e92b0a2014-05-15 09:35:06104 ss << '}';
105 return ss.str();
106}
pbos@webrtc.org29d58392013-05-16 12:08:03107
Peter Boströme4499152016-02-05 10:13:28108namespace {
109
Peter Boström39593972016-02-15 10:27:15110VideoCodecType PayloadNameToCodecType(const std::string& payload_name) {
111 if (payload_name == "VP8")
112 return kVideoCodecVP8;
113 if (payload_name == "VP9")
114 return kVideoCodecVP9;
115 if (payload_name == "H264")
116 return kVideoCodecH264;
117 return kVideoCodecGeneric;
118}
119
120bool PayloadTypeSupportsSkippingFecPackets(const std::string& payload_name) {
121 switch (PayloadNameToCodecType(payload_name)) {
122 case kVideoCodecVP8:
123 case kVideoCodecVP9:
124 return true;
125 case kVideoCodecH264:
126 case kVideoCodecGeneric:
127 return false;
128 case kVideoCodecI420:
129 case kVideoCodecRED:
130 case kVideoCodecULPFEC:
131 case kVideoCodecUnknown:
132 RTC_NOTREACHED();
133 return false;
134 }
135 RTC_NOTREACHED();
136 return false;
137}
138
Peter Boström23353ab2016-02-24 14:19:55139// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
140// pipelining encoders better (multiple input frames before something comes
141// out). This should effectively turn off CPU adaptations for systems that
142// remotely cope with the load right now.
Peter Boströme4499152016-02-05 10:13:28143CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) {
144 CpuOveruseOptions options;
145 if (full_overuse_time) {
Peter Boström23353ab2016-02-24 14:19:55146 options.low_encode_usage_threshold_percent = 150;
147 options.high_encode_usage_threshold_percent = 200;
Peter Boströme4499152016-02-05 10:13:28148 }
149 return options;
150}
151} // namespace
152
pbos@webrtc.org024e4d52014-05-15 10:03:24153namespace internal {
pbos@webrtc.org2bb1bda2014-07-07 13:06:48154VideoSendStream::VideoSendStream(
Peter Boström45553ae2015-05-08 11:54:38155 int num_cpu_cores,
Peter Boströmf16fcbe2015-04-30 10:16:05156 ProcessThread* module_process_thread,
mflodmane3787022015-10-21 11:24:28157 CallStats* call_stats,
mflodman0c478b32015-10-21 13:52:16158 CongestionController* congestion_controller,
Stefan Holmer58c664c2016-02-08 13:31:30159 VieRemb* remb,
mflodman0e7e2592015-11-13 05:02:42160 BitrateAllocator* bitrate_allocator,
pbos@webrtc.org2bb1bda2014-07-07 13:06:48161 const VideoSendStream::Config& config,
pbos@webrtc.orgbbe0a852014-09-19 12:30:25162 const VideoEncoderConfig& encoder_config,
Peter Boström45553ae2015-05-08 11:54:38163 const std::map<uint32_t, RtpState>& suspended_ssrcs)
sprangb4a1ae52015-12-03 16:10:08164 : stats_proxy_(Clock::GetRealTimeClock(),
165 config,
166 encoder_config.content_type),
sprang@webrtc.org40709352013-11-26 11:41:59167 encoded_frame_proxy_(config.post_encode_callback),
pbos@webrtc.org64887612013-11-14 08:58:14168 config_(config),
pbos@webrtc.org2bb1bda2014-07-07 13:06:48169 suspended_ssrcs_(suspended_ssrcs),
Peter Boströmf16fcbe2015-04-30 10:16:05170 module_process_thread_(module_process_thread),
mflodmane3787022015-10-21 11:24:28171 call_stats_(call_stats),
mflodman0c478b32015-10-21 13:52:16172 congestion_controller_(congestion_controller),
Stefan Holmer58c664c2016-02-08 13:31:30173 remb_(remb),
Peter Boströme4499152016-02-05 10:13:28174 overuse_detector_(
175 Clock::GetRealTimeClock(),
176 GetCpuOveruseOptions(config.encoder_settings.full_overuse_time),
177 this,
178 config.post_encode_callback,
179 &stats_proxy_),
Peter Boström8c66a002016-02-11 12:51:10180 vie_channel_(config.send_transport,
181 module_process_thread_,
182 &payload_router_,
183 nullptr,
Peter Boström45c44f02016-02-19 16:36:01184 &encoder_feedback_,
Peter Boström8c66a002016-02-11 12:51:10185 congestion_controller_->GetBitrateController()
186 ->CreateRtcpBandwidthObserver(),
187 congestion_controller_->GetTransportFeedbackObserver(),
188 nullptr,
189 call_stats_->rtcp_rtt_stats(),
190 congestion_controller_->pacer(),
191 congestion_controller_->packet_router(),
192 config_.rtp.ssrcs.size(),
193 true),
Peter Boström1f7d77f2016-02-11 14:30:14194 vie_receiver_(vie_channel_.vie_receiver()),
Peter Boström579e8322016-02-12 15:30:04195 vie_encoder_(num_cpu_cores,
Peter Boström0013dcc2016-02-19 19:42:19196 config_.rtp.ssrcs,
Peter Boström579e8322016-02-12 15:30:04197 module_process_thread_,
198 &stats_proxy_,
199 config.pre_encode_callback,
200 &overuse_detector_,
201 congestion_controller_->pacer(),
202 &payload_router_,
203 bitrate_allocator),
204 vcm_(vie_encoder_.vcm()),
Peter Boström723ead82016-02-22 14:14:01205 rtp_rtcp_modules_(vie_channel_.rtp_rtcp()),
Peter Boström8c66a002016-02-11 12:51:10206 input_(&vie_encoder_,
207 config_.local_renderer,
208 &stats_proxy_,
209 &overuse_detector_) {
pbosa2f30de2015-10-15 12:22:13210 LOG(LS_INFO) << "VideoSendStream: " << config_.ToString();
Stefan Holmer58c664c2016-02-08 13:31:30211
henrikg91d6ede2015-09-17 07:24:34212 RTC_DCHECK(!config_.rtp.ssrcs.empty());
Stefan Holmer58c664c2016-02-08 13:31:30213 RTC_DCHECK(module_process_thread_);
214 RTC_DCHECK(call_stats_);
215 RTC_DCHECK(congestion_controller_);
216 RTC_DCHECK(remb_);
mflodman949c2f02015-10-16 09:31:11217
Peter Boström8c66a002016-02-11 12:51:10218 RTC_CHECK(vie_encoder_.Init());
Peter Boström45c44f02016-02-19 16:36:01219 encoder_feedback_.Init(config_.rtp.ssrcs, &vie_encoder_);
Peter Boström8c66a002016-02-11 12:51:10220 RTC_CHECK(vie_channel_.Init() == 0);
mflodman949c2f02015-10-16 09:31:11221
Peter Boström8c66a002016-02-11 12:51:10222 vcm_->RegisterProtectionCallback(vie_channel_.vcm_protection_callback());
mflodman949c2f02015-10-16 09:31:11223
Peter Boström8c66a002016-02-11 12:51:10224 call_stats_->RegisterStatsObserver(vie_channel_.GetStatsObserver());
mflodman949c2f02015-10-16 09:31:11225
pbos@webrtc.org29023282013-09-11 10:14:56226 for (size_t i = 0; i < config_.rtp.extensions.size(); ++i) {
227 const std::string& extension = config_.rtp.extensions[i].name;
228 int id = config_.rtp.extensions[i].id;
Peter Boström23914fe2015-03-31 13:08:04229 // One-byte-extension local identifiers are in the range 1-14 inclusive.
henrikg91d6ede2015-09-17 07:24:34230 RTC_DCHECK_GE(id, 1);
231 RTC_DCHECK_LE(id, 14);
Peter Boström9c017252016-02-26 15:26:20232 RTC_DCHECK(RtpExtension::IsSupportedForVideo(extension));
233 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
234 RTC_CHECK_EQ(0, rtp_rtcp->RegisterSendRtpHeaderExtension(
235 StringToRtpExtensionType(extension), id));
pbos@webrtc.org29023282013-09-11 10:14:56236 }
237 }
pbos@webrtc.org29d58392013-05-16 12:08:03238
Peter Boström723ead82016-02-22 14:14:01239 remb_->AddRembSender(rtp_rtcp_modules_[0]);
240 rtp_rtcp_modules_[0]->SetREMBStatus(true);
mflodman@webrtc.org92c27932013-12-13 16:36:28241
pbos@webrtc.org0e63e762013-09-20 11:56:26242 // Enable NACK, FEC or both.
pbosba8c15b2015-07-14 16:36:34243 const bool enable_protection_nack = config_.rtp.nack.rtp_history_ms > 0;
Peter Boström39593972016-02-15 10:27:15244 bool enable_protection_fec = config_.rtp.fec.red_payload_type != -1;
245 // Payload types without picture ID cannot determine that a stream is complete
246 // without retransmitting FEC, so using FEC + NACK for H.264 (for instance) is
247 // a waste of bandwidth since FEC packets still have to be transmitted. Note
248 // that this is not the case with FLEXFEC.
249 if (enable_protection_nack &&
250 !PayloadTypeSupportsSkippingFecPackets(
251 config_.encoder_settings.payload_name)) {
252 LOG(LS_WARNING) << "Transmitting payload type without picture ID using"
253 "NACK+FEC is a waste of bandwidth since FEC packets "
254 "also have to be retransmitted. Disabling FEC.";
255 enable_protection_fec = false;
256 }
pbosba8c15b2015-07-14 16:36:34257 // TODO(changbin): Should set RTX for RED mapping in RTP sender in future.
Peter Boström8c66a002016-02-11 12:51:10258 vie_channel_.SetProtectionMode(enable_protection_nack, enable_protection_fec,
Peter Boström723ead82016-02-22 14:14:01259 config_.rtp.fec.red_payload_type,
260 config_.rtp.fec.ulpfec_payload_type);
Peter Boström8c66a002016-02-11 12:51:10261 vie_encoder_.SetProtectionMethod(enable_protection_nack,
Peter Boström723ead82016-02-22 14:14:01262 enable_protection_fec);
pbos@webrtc.org0e63e762013-09-20 11:56:26263
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09264 ConfigureSsrcs();
265
Peter Boström723ead82016-02-22 14:14:01266 // TODO(pbos): Should we set CNAME on all RTP modules?
267 rtp_rtcp_modules_.front()->SetCNAME(config_.rtp.c_name.c_str());
sprang@webrtc.org25fce9a2013-10-16 13:29:14268 // 28 to match packet overhead in ModuleRtpRtcpImpl.
Peter Boström723ead82016-02-22 14:14:01269 static const size_t kRtpPacketSizeOverhead = 28;
270 RTC_DCHECK_LE(config_.rtp.max_packet_size, 0xFFFFu + kRtpPacketSizeOverhead);
271 const uint16_t mtu = static_cast<uint16_t>(config_.rtp.max_packet_size +
272 kRtpPacketSizeOverhead);
273 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
274 rtp_rtcp->RegisterRtcpStatisticsCallback(&stats_proxy_);
275 rtp_rtcp->RegisterSendChannelRtpStatisticsCallback(&stats_proxy_);
276 rtp_rtcp->SetMaxTransferUnit(mtu);
277 }
pbos@webrtc.org29d58392013-05-16 12:08:03278
henrikg91d6ede2015-09-17 07:24:34279 RTC_DCHECK(config.encoder_settings.encoder != nullptr);
280 RTC_DCHECK_GE(config.encoder_settings.payload_type, 0);
281 RTC_DCHECK_LE(config.encoder_settings.payload_type, 127);
Peter Boström8c66a002016-02-11 12:51:10282 RTC_CHECK_EQ(0, vie_encoder_.RegisterExternalEncoder(
henrikg91d6ede2015-09-17 07:24:34283 config.encoder_settings.encoder,
284 config.encoder_settings.payload_type,
285 config.encoder_settings.internal_source));
stefan@webrtc.org360e3762013-08-22 09:29:56286
henrikg91d6ede2015-09-17 07:24:34287 RTC_CHECK(ReconfigureVideoEncoder(encoder_config));
pbos@webrtc.orgfe1ef932013-10-21 10:34:43288
Peter Boström8c66a002016-02-11 12:51:10289 vie_channel_.RegisterSendSideDelayObserver(&stats_proxy_);
stefan@webrtc.org168f23f2014-07-11 13:44:02290
Peter Boström94cc1fe2015-04-29 12:08:41291 if (config_.post_encode_callback)
Peter Boström8c66a002016-02-11 12:51:10292 vie_encoder_.RegisterPostEncodeImageCallback(&encoded_frame_proxy_);
henrik.lundin@webrtc.orgba975e22013-10-23 11:04:57293
Peter Boströme23e7372015-10-08 09:44:14294 if (config_.suspend_below_min_bitrate)
Peter Boström8c66a002016-02-11 12:51:10295 vie_encoder_.SuspendBelowMinBitrate();
sprang@webrtc.orgccd42842014-01-07 09:54:34296
Peter Boström8c66a002016-02-11 12:51:10297 vie_channel_.RegisterRtcpPacketTypeCounterObserver(&stats_proxy_);
298 vie_channel_.RegisterSendBitrateObserver(&stats_proxy_);
299 vie_channel_.RegisterSendFrameCountObserver(&stats_proxy_);
Peter Boströme4499152016-02-05 10:13:28300
301 module_process_thread_->RegisterModule(&overuse_detector_);
pbos@webrtc.org29d58392013-05-16 12:08:03302}
303
304VideoSendStream::~VideoSendStream() {
pbosa2f30de2015-10-15 12:22:13305 LOG(LS_INFO) << "~VideoSendStream: " << config_.ToString();
Peter Boströmca835252016-02-11 14:59:46306 Stop();
307
Peter Boströme4499152016-02-05 10:13:28308 module_process_thread_->DeRegisterModule(&overuse_detector_);
Peter Boström8c66a002016-02-11 12:51:10309 vie_channel_.RegisterSendFrameCountObserver(nullptr);
310 vie_channel_.RegisterSendBitrateObserver(nullptr);
311 vie_channel_.RegisterRtcpPacketTypeCounterObserver(nullptr);
sprang@webrtc.orgccd42842014-01-07 09:54:34312
Peter Boström723ead82016-02-22 14:14:01313 vie_encoder_.DeRegisterExternalEncoder(config_.encoder_settings.payload_type);
stefan@webrtc.org360e3762013-08-22 09:29:56314
Peter Boström8c66a002016-02-11 12:51:10315 call_stats_->DeregisterStatsObserver(vie_channel_.GetStatsObserver());
Peter Boström723ead82016-02-22 14:14:01316 rtp_rtcp_modules_[0]->SetREMBStatus(false);
317 remb_->RemoveRembSender(rtp_rtcp_modules_[0]);
mflodman949c2f02015-10-16 09:31:11318
Peter Boström45c44f02016-02-19 16:36:01319 // ViEChannel outlives ViEEncoder so remove encoder from feedback before
320 // destruction.
321 encoder_feedback_.TearDown();
mflodman949c2f02015-10-16 09:31:11322
mflodman0c478b32015-10-21 13:52:16323 congestion_controller_->GetRemoteBitrateEstimator(false)->RemoveStream(
Peter Boström1f7d77f2016-02-11 14:30:14324 vie_receiver_->GetRemoteSsrc());
pbos@webrtc.org29d58392013-05-16 12:08:03325}
326
Peter Boström4b91bd02015-06-26 04:58:16327VideoCaptureInput* VideoSendStream::Input() {
Peter Boström8c66a002016-02-11 12:51:10328 return &input_;
pbos@webrtc.org29d58392013-05-16 12:08:03329}
330
pbos@webrtc.orga5c8d2c2014-04-24 11:13:21331void VideoSendStream::Start() {
Peter Boström8c66a002016-02-11 12:51:10332 vie_encoder_.Pause();
333 if (vie_channel_.StartSend() == 0) {
Peter Boström45553ae2015-05-08 11:54:38334 // Was not already started, trigger a keyframe.
Peter Boström8c66a002016-02-11 12:51:10335 vie_encoder_.SendKeyFrame();
Peter Boström45553ae2015-05-08 11:54:38336 }
Peter Boström8c66a002016-02-11 12:51:10337 vie_encoder_.Restart();
Peter Boströmca835252016-02-11 14:59:46338 vie_receiver_->StartReceive();
pbos@webrtc.org29d58392013-05-16 12:08:03339}
340
pbos@webrtc.orga5c8d2c2014-04-24 11:13:21341void VideoSendStream::Stop() {
Peter Boström45553ae2015-05-08 11:54:38342 // TODO(pbos): Make sure the encoder stops here.
Peter Boström8c66a002016-02-11 12:51:10343 vie_channel_.StopSend();
Peter Boströmca835252016-02-11 14:59:46344 vie_receiver_->StopReceive();
pbos@webrtc.org29d58392013-05-16 12:08:03345}
346
pbos@webrtc.orgf577ae92014-03-19 08:43:57347bool VideoSendStream::ReconfigureVideoEncoder(
pbos@webrtc.orgbbe0a852014-09-19 12:30:25348 const VideoEncoderConfig& config) {
pbos@webrtc.org50fe3592015-01-29 12:33:07349 TRACE_EVENT0("webrtc", "VideoSendStream::(Re)configureVideoEncoder");
pbos@webrtc.orgad3b5a52014-10-24 09:23:21350 LOG(LS_INFO) << "(Re)configureVideoEncoder: " << config.ToString();
pbos@webrtc.orgbbe0a852014-09-19 12:30:25351 const std::vector<VideoStream>& streams = config.streams;
henrikg91d6ede2015-09-17 07:24:34352 RTC_DCHECK(!streams.empty());
353 RTC_DCHECK_GE(config_.rtp.ssrcs.size(), streams.size());
pbos@webrtc.org64887612013-11-14 08:58:14354
pbos@webrtc.orgf577ae92014-03-19 08:43:57355 VideoCodec video_codec;
356 memset(&video_codec, 0, sizeof(video_codec));
Peter Boström39593972016-02-15 10:27:15357 video_codec.codecType =
358 PayloadNameToCodecType(config_.encoder_settings.payload_name);
pbos@webrtc.orgb7ed7792014-10-31 13:08:10359
pbos@webrtc.orgbbe0a852014-09-19 12:30:25360 switch (config.content_type) {
Erik Språng143cec12015-04-28 08:01:41361 case VideoEncoderConfig::ContentType::kRealtimeVideo:
pbos@webrtc.orgbbe0a852014-09-19 12:30:25362 video_codec.mode = kRealtimeVideo;
363 break;
Erik Språng143cec12015-04-28 08:01:41364 case VideoEncoderConfig::ContentType::kScreen:
pbos@webrtc.orgbbe0a852014-09-19 12:30:25365 video_codec.mode = kScreensharing;
pbos@webrtc.orgb7ed7792014-10-31 13:08:10366 if (config.streams.size() == 1 &&
367 config.streams[0].temporal_layer_thresholds_bps.size() == 1) {
368 video_codec.targetBitrate =
369 config.streams[0].temporal_layer_thresholds_bps[0] / 1000;
370 }
pbos@webrtc.orgbbe0a852014-09-19 12:30:25371 break;
372 }
pbos@webrtc.orgf577ae92014-03-19 08:43:57373
374 if (video_codec.codecType == kVideoCodecVP8) {
pbos@webrtc.org6cd6ba82014-09-18 12:42:28375 video_codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings();
marpan@webrtc.org5b883172014-11-01 06:10:48376 } else if (video_codec.codecType == kVideoCodecVP9) {
377 video_codec.codecSpecific.VP9 = VideoEncoder::GetDefaultVp9Settings();
stefan@webrtc.org79c33592014-08-06 09:24:53378 } else if (video_codec.codecType == kVideoCodecH264) {
pbos@webrtc.org6cd6ba82014-09-18 12:42:28379 video_codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings();
pbos@webrtc.orgf577ae92014-03-19 08:43:57380 }
381
pbos@webrtc.org91f17522014-07-10 10:13:37382 if (video_codec.codecType == kVideoCodecVP8) {
pbos@webrtc.org2b4ce3a2015-03-23 13:12:24383 if (config.encoder_specific_settings != nullptr) {
pbos@webrtc.orgbbe0a852014-09-19 12:30:25384 video_codec.codecSpecific.VP8 = *reinterpret_cast<const VideoCodecVP8*>(
385 config.encoder_specific_settings);
pbos@webrtc.org91f17522014-07-10 10:13:37386 }
pbos@webrtc.org759982d2014-09-22 09:32:46387 video_codec.codecSpecific.VP8.numberOfTemporalLayers =
pbos@webrtc.orgb7ed7792014-10-31 13:08:10388 static_cast<unsigned char>(
389 streams.back().temporal_layer_thresholds_bps.size() + 1);
pbos@webrtc.orgb9557a92015-03-20 19:52:56390 } else if (video_codec.codecType == kVideoCodecVP9) {
pbos@webrtc.org2b4ce3a2015-03-23 13:12:24391 if (config.encoder_specific_settings != nullptr) {
pbos@webrtc.orgb9557a92015-03-20 19:52:56392 video_codec.codecSpecific.VP9 = *reinterpret_cast<const VideoCodecVP9*>(
393 config.encoder_specific_settings);
philipelcfc319b2015-11-10 15:17:23394 if (video_codec.mode == kScreensharing) {
395 video_codec.codecSpecific.VP9.flexibleMode = true;
396 // For now VP9 screensharing use 1 temporal and 2 spatial layers.
397 RTC_DCHECK_EQ(video_codec.codecSpecific.VP9.numberOfTemporalLayers, 1);
398 RTC_DCHECK_EQ(video_codec.codecSpecific.VP9.numberOfSpatialLayers, 2);
399 }
pbos@webrtc.orgb9557a92015-03-20 19:52:56400 }
401 video_codec.codecSpecific.VP9.numberOfTemporalLayers =
402 static_cast<unsigned char>(
403 streams.back().temporal_layer_thresholds_bps.size() + 1);
404 } else if (video_codec.codecType == kVideoCodecH264) {
pbos@webrtc.org2b4ce3a2015-03-23 13:12:24405 if (config.encoder_specific_settings != nullptr) {
pbos@webrtc.orgb9557a92015-03-20 19:52:56406 video_codec.codecSpecific.H264 = *reinterpret_cast<const VideoCodecH264*>(
407 config.encoder_specific_settings);
408 }
pbos@webrtc.org91f17522014-07-10 10:13:37409 } else {
410 // TODO(pbos): Support encoder_settings codec-agnostically.
henrikg91d6ede2015-09-17 07:24:34411 RTC_DCHECK(config.encoder_specific_settings == nullptr)
pbos@webrtc.org2b4ce3a2015-03-23 13:12:24412 << "Encoder-specific settings for codec type not wired up.";
pbos@webrtc.org91f17522014-07-10 10:13:37413 }
414
pbos@webrtc.orgf577ae92014-03-19 08:43:57415 strncpy(video_codec.plName,
416 config_.encoder_settings.payload_name.c_str(),
417 kPayloadNameSize - 1);
418 video_codec.plName[kPayloadNameSize - 1] = '\0';
419 video_codec.plType = config_.encoder_settings.payload_type;
420 video_codec.numberOfSimulcastStreams =
421 static_cast<unsigned char>(streams.size());
422 video_codec.minBitrate = streams[0].min_bitrate_bps / 1000;
henrikg91d6ede2015-09-17 07:24:34423 RTC_DCHECK_LE(streams.size(), static_cast<size_t>(kMaxSimulcastStreams));
sprangce4aef12015-11-02 15:23:20424 if (video_codec.codecType == kVideoCodecVP9) {
425 // If the vector is empty, bitrates will be configured automatically.
426 RTC_DCHECK(config.spatial_layers.empty() ||
427 config.spatial_layers.size() ==
428 video_codec.codecSpecific.VP9.numberOfSpatialLayers);
429 RTC_DCHECK_LE(video_codec.codecSpecific.VP9.numberOfSpatialLayers,
430 kMaxSimulcastStreams);
431 for (size_t i = 0; i < config.spatial_layers.size(); ++i)
432 video_codec.spatialLayers[i] = config.spatial_layers[i];
433 }
pbos@webrtc.orgf577ae92014-03-19 08:43:57434 for (size_t i = 0; i < streams.size(); ++i) {
435 SimulcastStream* sim_stream = &video_codec.simulcastStream[i];
henrikg91d6ede2015-09-17 07:24:34436 RTC_DCHECK_GT(streams[i].width, 0u);
437 RTC_DCHECK_GT(streams[i].height, 0u);
438 RTC_DCHECK_GT(streams[i].max_framerate, 0);
pbos@webrtc.orgf577ae92014-03-19 08:43:57439 // Different framerates not supported per stream at the moment.
henrikg91d6ede2015-09-17 07:24:34440 RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate);
441 RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0);
442 RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps);
443 RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps);
444 RTC_DCHECK_GE(streams[i].max_qp, 0);
pbos@webrtc.orgf577ae92014-03-19 08:43:57445
mflodmand1590b22015-12-09 15:07:59446 sim_stream->width = static_cast<uint16_t>(streams[i].width);
447 sim_stream->height = static_cast<uint16_t>(streams[i].height);
pbos@webrtc.orgf577ae92014-03-19 08:43:57448 sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000;
449 sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000;
450 sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000;
451 sim_stream->qpMax = streams[i].max_qp;
pbos@webrtc.orgb7ed7792014-10-31 13:08:10452 sim_stream->numberOfTemporalLayers = static_cast<unsigned char>(
453 streams[i].temporal_layer_thresholds_bps.size() + 1);
pbos@webrtc.orgf577ae92014-03-19 08:43:57454
455 video_codec.width = std::max(video_codec.width,
mflodmand1590b22015-12-09 15:07:59456 static_cast<uint16_t>(streams[i].width));
pbos@webrtc.orgf577ae92014-03-19 08:43:57457 video_codec.height = std::max(
mflodmand1590b22015-12-09 15:07:59458 video_codec.height, static_cast<uint16_t>(streams[i].height));
pbos@webrtc.orgf577ae92014-03-19 08:43:57459 video_codec.minBitrate =
mflodmand1590b22015-12-09 15:07:59460 std::min(static_cast<uint16_t>(video_codec.minBitrate),
461 static_cast<uint16_t>(streams[i].min_bitrate_bps / 1000));
pbos@webrtc.orgf577ae92014-03-19 08:43:57462 video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000;
463 video_codec.qpMax = std::max(video_codec.qpMax,
464 static_cast<unsigned int>(streams[i].max_qp));
465 }
Stefan Holmere5904162015-03-26 10:11:06466
467 // Set to zero to not update the bitrate controller from ViEEncoder, as
468 // the bitrate controller is already set from Call.
469 video_codec.startBitrate = 0;
pbos@webrtc.orgf577ae92014-03-19 08:43:57470
henrikg91d6ede2015-09-17 07:24:34471 RTC_DCHECK_GT(streams[0].max_framerate, 0);
pbos@webrtc.org6ae48c62014-06-06 10:49:19472 video_codec.maxFramerate = streams[0].max_framerate;
pbos@webrtc.orgf577ae92014-03-19 08:43:57473
Peter Boström5cb9ce42015-05-05 13:16:30474 if (!SetSendCodec(video_codec))
pbos@webrtc.org32452b22014-10-22 12:15:24475 return false;
476
Peter Boström20f3f942015-05-15 09:33:39477 // Clear stats for disabled layers.
478 for (size_t i = video_codec.numberOfSimulcastStreams;
479 i < config_.rtp.ssrcs.size(); ++i) {
480 stats_proxy_.OnInactiveSsrc(config_.rtp.ssrcs[i]);
481 }
482
sprangb4a1ae52015-12-03 16:10:08483 stats_proxy_.SetContentType(config.content_type);
484
henrikg91d6ede2015-09-17 07:24:34485 RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0);
Peter Boström8c66a002016-02-11 12:51:10486 vie_encoder_.SetMinTransmitBitrate(config.min_transmit_bitrate_bps / 1000);
pbos@webrtc.orgad3b5a52014-10-24 09:23:21487
pbos@webrtc.org00873182014-11-25 14:03:34488 encoder_config_ = config;
pbos@webrtc.org32452b22014-10-22 12:15:24489 return true;
pbos@webrtc.org29d58392013-05-16 12:08:03490}
491
pbos@webrtc.orgbbb07e62013-08-05 12:01:36492bool VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
Peter Boström1f7d77f2016-02-11 14:30:14493 return vie_receiver_->DeliverRtcp(packet, length);
pbos@webrtc.orgbbb07e62013-08-05 12:01:36494}
sprang@webrtc.orgccd42842014-01-07 09:54:34495
pbos@webrtc.org273a4142014-12-01 15:23:21496VideoSendStream::Stats VideoSendStream::GetStats() {
stefan@webrtc.org168f23f2014-07-11 13:44:02497 return stats_proxy_.GetStats();
sprang@webrtc.orgccd42842014-01-07 09:54:34498}
499
solenberge5269742015-09-08 12:13:22500void VideoSendStream::OveruseDetected() {
501 if (config_.overuse_callback)
502 config_.overuse_callback->OnLoadUpdate(LoadObserver::kOveruse);
503}
504
505void VideoSendStream::NormalUsage() {
506 if (config_.overuse_callback)
507 config_.overuse_callback->OnLoadUpdate(LoadObserver::kUnderuse);
508}
509
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09510void VideoSendStream::ConfigureSsrcs() {
Peter Boström723ead82016-02-22 14:14:01511 // Configure regular SSRCs.
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09512 for (size_t i = 0; i < config_.rtp.ssrcs.size(); ++i) {
513 uint32_t ssrc = config_.rtp.ssrcs[i];
Peter Boström723ead82016-02-22 14:14:01514 RtpRtcp* const rtp_rtcp = rtp_rtcp_modules_[i];
515 rtp_rtcp->SetSSRC(ssrc);
516
517 // Restore RTP state if previous existed.
pbos@webrtc.org2bb1bda2014-07-07 13:06:48518 RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc);
519 if (it != suspended_ssrcs_.end())
Peter Boström723ead82016-02-22 14:14:01520 rtp_rtcp->SetRtpStateForSsrc(ssrc, it->second);
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09521 }
522
Peter Boström723ead82016-02-22 14:14:01523 // Set up RTX if available.
524 if (config_.rtp.rtx.ssrcs.empty())
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09525 return;
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09526
Peter Boström723ead82016-02-22 14:14:01527 // Configure RTX SSRCs.
henrikg91d6ede2015-09-17 07:24:34528 RTC_DCHECK_EQ(config_.rtp.rtx.ssrcs.size(), config_.rtp.ssrcs.size());
pbos@webrtc.org2bb1bda2014-07-07 13:06:48529 for (size_t i = 0; i < config_.rtp.rtx.ssrcs.size(); ++i) {
530 uint32_t ssrc = config_.rtp.rtx.ssrcs[i];
Peter Boström723ead82016-02-22 14:14:01531 RtpRtcp* const rtp_rtcp = rtp_rtcp_modules_[i];
532 rtp_rtcp->SetRtxSsrc(ssrc);
pbos@webrtc.org2bb1bda2014-07-07 13:06:48533 RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc);
534 if (it != suspended_ssrcs_.end())
Peter Boström723ead82016-02-22 14:14:01535 rtp_rtcp->SetRtpStateForSsrc(ssrc, it->second);
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09536 }
537
Peter Boström723ead82016-02-22 14:14:01538 // Configure RTX payload types.
henrikg91d6ede2015-09-17 07:24:34539 RTC_DCHECK_GE(config_.rtp.rtx.payload_type, 0);
Peter Boström723ead82016-02-22 14:14:01540 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
541 rtp_rtcp->SetRtxSendPayloadType(config_.rtp.rtx.payload_type,
542 config_.encoder_settings.payload_type);
543 rtp_rtcp->SetRtxSendStatus(kRtxRetransmitted | kRtxRedundantPayloads);
544 }
Stefan Holmer10880012016-02-03 12:29:59545 if (config_.rtp.fec.red_payload_type != -1 &&
546 config_.rtp.fec.red_rtx_payload_type != -1) {
Peter Boström723ead82016-02-22 14:14:01547 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
548 rtp_rtcp->SetRtxSendPayloadType(config_.rtp.fec.red_rtx_payload_type,
549 config_.rtp.fec.red_payload_type);
550 }
Stefan Holmer10880012016-02-03 12:29:59551 }
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09552}
553
pbos@webrtc.org2bb1bda2014-07-07 13:06:48554std::map<uint32_t, RtpState> VideoSendStream::GetRtpStates() const {
555 std::map<uint32_t, RtpState> rtp_states;
556 for (size_t i = 0; i < config_.rtp.ssrcs.size(); ++i) {
557 uint32_t ssrc = config_.rtp.ssrcs[i];
Peter Boström8c66a002016-02-11 12:51:10558 rtp_states[ssrc] = vie_channel_.GetRtpStateForSsrc(ssrc);
pbos@webrtc.org2bb1bda2014-07-07 13:06:48559 }
560
561 for (size_t i = 0; i < config_.rtp.rtx.ssrcs.size(); ++i) {
562 uint32_t ssrc = config_.rtp.rtx.ssrcs[i];
Peter Boström8c66a002016-02-11 12:51:10563 rtp_states[ssrc] = vie_channel_.GetRtpStateForSsrc(ssrc);
pbos@webrtc.org2bb1bda2014-07-07 13:06:48564 }
565
566 return rtp_states;
567}
568
Jelena Marusiccd670222015-07-16 07:30:09569void VideoSendStream::SignalNetworkState(NetworkState state) {
pbos@webrtc.org26c0c412014-09-03 16:17:12570 // When network goes up, enable RTCP status before setting transmission state.
571 // When it goes down, disable RTCP afterwards. This ensures that any packets
572 // sent due to the network state changed will not be dropped.
Peter Boström723ead82016-02-22 14:14:01573 if (state == kNetworkUp) {
574 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_)
575 rtp_rtcp->SetRTCPStatus(config_.rtp.rtcp_mode);
576 }
Peter Boström8c66a002016-02-11 12:51:10577 vie_encoder_.SetNetworkTransmissionState(state == kNetworkUp);
Peter Boström723ead82016-02-22 14:14:01578 if (state == kNetworkDown) {
579 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_)
580 rtp_rtcp->SetRTCPStatus(RtcpMode::kOff);
581 }
stefan@webrtc.org0bae1fa2014-11-05 14:05:29582}
pbos@webrtc.org2b19f062014-12-11 13:26:09583
mflodman0e7e2592015-11-13 05:02:42584int VideoSendStream::GetPaddingNeededBps() const {
Peter Boström8c66a002016-02-11 12:51:10585 return vie_encoder_.GetPaddingNeededBps();
mflodman0e7e2592015-11-13 05:02:42586}
587
Peter Boström5cb9ce42015-05-05 13:16:30588bool VideoSendStream::SetSendCodec(VideoCodec video_codec) {
mflodmanc4a1c372015-11-06 12:33:51589 static const int kEncoderMinBitrate = 30;
Peter Boström5cb9ce42015-05-05 13:16:30590 if (video_codec.maxBitrate == 0) {
591 // Unset max bitrate -> cap to one bit per pixel.
592 video_codec.maxBitrate =
593 (video_codec.width * video_codec.height * video_codec.maxFramerate) /
594 1000;
595 }
596
mflodmanc4a1c372015-11-06 12:33:51597 if (video_codec.minBitrate < kEncoderMinBitrate)
598 video_codec.minBitrate = kEncoderMinBitrate;
599 if (video_codec.maxBitrate < kEncoderMinBitrate)
600 video_codec.maxBitrate = kEncoderMinBitrate;
Peter Boström5cb9ce42015-05-05 13:16:30601
602 // Stop the media flow while reconfiguring.
Peter Boström8c66a002016-02-11 12:51:10603 vie_encoder_.Pause();
Peter Boström5cb9ce42015-05-05 13:16:30604
Peter Boström8c66a002016-02-11 12:51:10605 if (vie_encoder_.SetEncoder(video_codec) != 0) {
Peter Boström5cb9ce42015-05-05 13:16:30606 LOG(LS_ERROR) << "Failed to set encoder.";
607 return false;
608 }
609
Peter Boström8c66a002016-02-11 12:51:10610 if (vie_channel_.SetSendCodec(video_codec, false) != 0) {
Peter Boström5cb9ce42015-05-05 13:16:30611 LOG(LS_ERROR) << "Failed to set send codec.";
612 return false;
613 }
614
Peter Boström5cb9ce42015-05-05 13:16:30615 // Restart the media flow
Peter Boström8c66a002016-02-11 12:51:10616 vie_encoder_.Restart();
Peter Boström5cb9ce42015-05-05 13:16:30617
618 return true;
619}
pbos@webrtc.org29d58392013-05-16 12:08:03620} // namespace internal
621} // namespace webrtc