Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | #include "test/scenario/video_stream.h" |
| 11 | |
| 12 | #include <algorithm> |
Mirko Bonadei | 317a1f0 | 2019-09-17 15:06:18 | [diff] [blame] | 13 | #include <memory> |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 14 | #include <utility> |
| 15 | |
Danil Chapovalov | 99b71df | 2018-10-26 13:57:48 | [diff] [blame] | 16 | #include "api/test/video/function_video_encoder_factory.h" |
Jiawei Ou | c2ebe21 | 2018-11-08 18:02:56 | [diff] [blame] | 17 | #include "api/video/builtin_video_bitrate_allocator_factory.h" |
Steve Anton | 10542f2 | 2019-01-11 17:11:00 | [diff] [blame] | 18 | #include "media/base/media_constants.h" |
| 19 | #include "media/engine/internal_decoder_factory.h" |
| 20 | #include "media/engine/internal_encoder_factory.h" |
| 21 | #include "media/engine/webrtc_video_engine.h" |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 22 | #include "test/call_test.h" |
| 23 | #include "test/fake_encoder.h" |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 24 | #include "test/scenario/hardware_codecs.h" |
Steve Anton | 10542f2 | 2019-01-11 17:11:00 | [diff] [blame] | 25 | #include "test/testsupport/file_utils.h" |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 26 | |
| 27 | namespace webrtc { |
| 28 | namespace test { |
| 29 | namespace { |
Elad Alon | d8d3248 | 2019-02-18 22:45:57 | [diff] [blame] | 30 | enum : int { // The first valid value is 1. |
| 31 | kTransportSequenceNumberExtensionId = 1, |
Sebastian Jansson | e112bb8 | 2019-06-13 15:36:01 | [diff] [blame] | 32 | kAbsSendTimeExtensionId, |
Elad Alon | d8d3248 | 2019-02-18 22:45:57 | [diff] [blame] | 33 | kVideoContentTypeExtensionId, |
| 34 | kVideoRotationRtpExtensionId, |
| 35 | }; |
| 36 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 37 | constexpr int kDefaultMaxQp = cricket::WebRtcVideoChannel::kDefaultQpMax; |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 38 | uint8_t CodecTypeToPayloadType(VideoCodecType codec_type) { |
| 39 | switch (codec_type) { |
| 40 | case VideoCodecType::kVideoCodecGeneric: |
| 41 | return CallTest::kFakeVideoSendPayloadType; |
| 42 | case VideoCodecType::kVideoCodecVP8: |
| 43 | return CallTest::kPayloadTypeVP8; |
| 44 | case VideoCodecType::kVideoCodecVP9: |
| 45 | return CallTest::kPayloadTypeVP9; |
| 46 | case VideoCodecType::kVideoCodecH264: |
| 47 | return CallTest::kPayloadTypeH264; |
| 48 | default: |
| 49 | RTC_NOTREACHED(); |
| 50 | } |
| 51 | return {}; |
| 52 | } |
| 53 | std::string CodecTypeToCodecName(VideoCodecType codec_type) { |
| 54 | switch (codec_type) { |
| 55 | case VideoCodecType::kVideoCodecGeneric: |
| 56 | return ""; |
| 57 | case VideoCodecType::kVideoCodecVP8: |
| 58 | return cricket::kVp8CodecName; |
| 59 | case VideoCodecType::kVideoCodecVP9: |
| 60 | return cricket::kVp9CodecName; |
| 61 | case VideoCodecType::kVideoCodecH264: |
| 62 | return cricket::kH264CodecName; |
| 63 | default: |
| 64 | RTC_NOTREACHED(); |
| 65 | } |
| 66 | return {}; |
| 67 | } |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 68 | VideoEncoderConfig::ContentType ConvertContentType( |
| 69 | VideoStreamConfig::Encoder::ContentType content_type) { |
| 70 | switch (content_type) { |
| 71 | case VideoStreamConfig::Encoder::ContentType::kVideo: |
| 72 | return VideoEncoderConfig::ContentType::kRealtimeVideo; |
| 73 | break; |
| 74 | case VideoStreamConfig::Encoder::ContentType::kScreen: |
| 75 | return VideoEncoderConfig::ContentType::kScreen; |
| 76 | } |
| 77 | } |
Sebastian Jansson | 06c5145 | 2019-02-18 15:55:15 | [diff] [blame] | 78 | InterLayerPredMode ToInterLayerPredMode( |
| 79 | VideoStreamConfig::Encoder::Layers::Prediction value) { |
| 80 | using Pred = VideoStreamConfig::Encoder::Layers::Prediction; |
| 81 | switch (value) { |
| 82 | case Pred::kTemporalOnly: |
| 83 | return InterLayerPredMode::kOff; |
| 84 | case Pred::kSpatialOnKey: |
| 85 | return InterLayerPredMode::kOnKeyPic; |
| 86 | case Pred::kFull: |
| 87 | return InterLayerPredMode::kOn; |
| 88 | } |
| 89 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 90 | std::vector<RtpExtension> GetVideoRtpExtensions( |
| 91 | const VideoStreamConfig config) { |
Sebastian Jansson | 24cf260 | 2019-06-07 13:24:57 | [diff] [blame] | 92 | std::vector<RtpExtension> res = { |
| 93 | RtpExtension(RtpExtension::kVideoContentTypeUri, |
| 94 | kVideoContentTypeExtensionId), |
| 95 | RtpExtension(RtpExtension::kVideoRotationUri, |
| 96 | kVideoRotationRtpExtensionId)}; |
| 97 | if (config.stream.packet_feedback) { |
| 98 | res.push_back(RtpExtension(RtpExtension::kTransportSequenceNumberUri, |
| 99 | kTransportSequenceNumberExtensionId)); |
| 100 | } |
Sebastian Jansson | e112bb8 | 2019-06-13 15:36:01 | [diff] [blame] | 101 | if (config.stream.abs_send_time) { |
| 102 | res.push_back( |
| 103 | RtpExtension(RtpExtension::kAbsSendTimeUri, kAbsSendTimeExtensionId)); |
| 104 | } |
Sebastian Jansson | 24cf260 | 2019-06-07 13:24:57 | [diff] [blame] | 105 | return res; |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 106 | } |
| 107 | |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 108 | std::string TransformFilePath(std::string path) { |
| 109 | static const std::string resource_prefix = "res://"; |
| 110 | int ext_pos = path.rfind("."); |
| 111 | if (ext_pos < 0) { |
| 112 | return test::ResourcePath(path, "yuv"); |
| 113 | } else if (path.find(resource_prefix) == 0) { |
| 114 | std::string name = path.substr(resource_prefix.length(), ext_pos); |
| 115 | std::string ext = path.substr(ext_pos, path.size()); |
| 116 | return test::ResourcePath(name, ext); |
| 117 | } |
| 118 | return path; |
| 119 | } |
| 120 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 121 | VideoSendStream::Config CreateVideoSendStreamConfig(VideoStreamConfig config, |
| 122 | std::vector<uint32_t> ssrcs, |
| 123 | Transport* send_transport) { |
| 124 | VideoSendStream::Config send_config(send_transport); |
| 125 | send_config.rtp.payload_name = CodecTypeToPayloadString(config.encoder.codec); |
| 126 | send_config.rtp.payload_type = CodecTypeToPayloadType(config.encoder.codec); |
Sebastian Jansson | c57b0ee | 2019-06-26 14:22:39 | [diff] [blame] | 127 | send_config.rtp.nack.rtp_history_ms = |
| 128 | config.stream.nack_history_time.ms<int>(); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 129 | |
| 130 | send_config.rtp.ssrcs = ssrcs; |
| 131 | send_config.rtp.extensions = GetVideoRtpExtensions(config); |
| 132 | |
| 133 | if (config.stream.use_flexfec) { |
| 134 | send_config.rtp.flexfec.payload_type = CallTest::kFlexfecPayloadType; |
| 135 | send_config.rtp.flexfec.ssrc = CallTest::kFlexfecSendSsrc; |
| 136 | send_config.rtp.flexfec.protected_media_ssrcs = ssrcs; |
| 137 | } |
| 138 | if (config.stream.use_ulpfec) { |
| 139 | send_config.rtp.ulpfec.red_payload_type = CallTest::kRedPayloadType; |
| 140 | send_config.rtp.ulpfec.ulpfec_payload_type = CallTest::kUlpfecPayloadType; |
| 141 | send_config.rtp.ulpfec.red_rtx_payload_type = CallTest::kRtxRedPayloadType; |
| 142 | } |
| 143 | return send_config; |
| 144 | } |
| 145 | rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings> |
Sebastian Jansson | 06c5145 | 2019-02-18 15:55:15 | [diff] [blame] | 146 | CreateVp9SpecificSettings(VideoStreamConfig video_config) { |
| 147 | constexpr auto kScreen = VideoStreamConfig::Encoder::ContentType::kScreen; |
| 148 | VideoStreamConfig::Encoder conf = video_config.encoder; |
| 149 | VideoCodecVP9 vp9 = VideoEncoder::GetDefaultVp9Settings(); |
| 150 | vp9.frameDroppingOn = conf.frame_dropping; |
| 151 | vp9.keyFrameInterval = conf.key_frame_interval.value_or(0); |
| 152 | vp9.numberOfTemporalLayers = static_cast<uint8_t>(conf.layers.temporal); |
| 153 | vp9.numberOfSpatialLayers = static_cast<uint8_t>(conf.layers.spatial); |
| 154 | vp9.interLayerPred = ToInterLayerPredMode(conf.layers.prediction); |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 155 | |
Sebastian Jansson | 06c5145 | 2019-02-18 15:55:15 | [diff] [blame] | 156 | if (conf.content_type == kScreen && |
| 157 | (video_config.source.framerate > 5 || conf.layers.spatial >= 3)) { |
| 158 | vp9.flexibleMode = true; |
| 159 | } |
| 160 | |
| 161 | if (conf.content_type == kScreen || |
| 162 | conf.layers.temporal * conf.layers.spatial) { |
| 163 | vp9.automaticResizeOn = false; |
| 164 | vp9.denoisingOn = false; |
| 165 | } else { |
| 166 | vp9.automaticResizeOn = conf.single.automatic_scaling; |
| 167 | vp9.denoisingOn = conf.single.denoising; |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 168 | } |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 169 | return new rtc::RefCountedObject< |
Sebastian Jansson | 06c5145 | 2019-02-18 15:55:15 | [diff] [blame] | 170 | VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9); |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 171 | } |
| 172 | |
| 173 | rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings> |
| 174 | CreateVp8SpecificSettings(VideoStreamConfig config) { |
| 175 | VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings(); |
| 176 | vp8_settings.frameDroppingOn = config.encoder.frame_dropping; |
| 177 | vp8_settings.keyFrameInterval = config.encoder.key_frame_interval.value_or(0); |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 178 | vp8_settings.numberOfTemporalLayers = config.encoder.layers.temporal; |
| 179 | if (config.encoder.layers.spatial * config.encoder.layers.temporal > 1) { |
| 180 | vp8_settings.automaticResizeOn = false; |
| 181 | vp8_settings.denoisingOn = false; |
| 182 | } else { |
| 183 | vp8_settings.automaticResizeOn = config.encoder.single.automatic_scaling; |
| 184 | vp8_settings.denoisingOn = config.encoder.single.denoising; |
| 185 | } |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 186 | return new rtc::RefCountedObject< |
| 187 | VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings); |
| 188 | } |
| 189 | |
| 190 | rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings> |
| 191 | CreateH264SpecificSettings(VideoStreamConfig config) { |
Sebastian Jansson | 06c5145 | 2019-02-18 15:55:15 | [diff] [blame] | 192 | RTC_DCHECK_EQ(config.encoder.layers.temporal, 1); |
| 193 | RTC_DCHECK_EQ(config.encoder.layers.spatial, 1); |
| 194 | |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 195 | VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings(); |
| 196 | h264_settings.frameDroppingOn = config.encoder.frame_dropping; |
| 197 | h264_settings.keyFrameInterval = |
| 198 | config.encoder.key_frame_interval.value_or(0); |
| 199 | return new rtc::RefCountedObject< |
| 200 | VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings); |
| 201 | } |
| 202 | |
| 203 | rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings> |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 204 | CreateEncoderSpecificSettings(VideoStreamConfig config) { |
| 205 | using Codec = VideoStreamConfig::Encoder::Codec; |
| 206 | switch (config.encoder.codec) { |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 207 | case Codec::kVideoCodecH264: |
| 208 | return CreateH264SpecificSettings(config); |
| 209 | case Codec::kVideoCodecVP8: |
| 210 | return CreateVp8SpecificSettings(config); |
| 211 | case Codec::kVideoCodecVP9: |
| 212 | return CreateVp9SpecificSettings(config); |
| 213 | case Codec::kVideoCodecGeneric: |
Danil Chapovalov | dc36829 | 2019-11-26 13:48:20 | [diff] [blame^] | 214 | case Codec::kVideoCodecAV1: |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 215 | return nullptr; |
| 216 | case Codec::kVideoCodecMultiplex: |
| 217 | RTC_NOTREACHED(); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 218 | return nullptr; |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | VideoEncoderConfig CreateVideoEncoderConfig(VideoStreamConfig config) { |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 223 | VideoEncoderConfig encoder_config; |
| 224 | encoder_config.codec_type = config.encoder.codec; |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 225 | encoder_config.content_type = ConvertContentType(config.encoder.content_type); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 226 | encoder_config.video_format = |
| 227 | SdpVideoFormat(CodecTypeToPayloadString(config.encoder.codec), {}); |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 228 | |
Sebastian Jansson | e822668 | 2019-04-30 12:29:09 | [diff] [blame] | 229 | encoder_config.number_of_streams = 1; |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 230 | if (config.encoder.codec == VideoStreamConfig::Encoder::Codec::kVideoCodecVP8) |
Sebastian Jansson | e822668 | 2019-04-30 12:29:09 | [diff] [blame] | 231 | encoder_config.number_of_streams = |
| 232 | static_cast<size_t>(config.encoder.layers.spatial); |
| 233 | encoder_config.simulcast_layers = |
| 234 | std::vector<VideoStream>(config.encoder.layers.spatial); |
Sebastian Jansson | 985ee68 | 2018-11-12 15:33:29 | [diff] [blame] | 235 | encoder_config.min_transmit_bitrate_bps = config.stream.pad_to_rate.bps(); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 236 | |
| 237 | std::string cricket_codec = CodecTypeToCodecName(config.encoder.codec); |
| 238 | if (!cricket_codec.empty()) { |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 239 | bool screenshare = config.encoder.content_type == |
| 240 | VideoStreamConfig::Encoder::ContentType::kScreen; |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 241 | encoder_config.video_stream_factory = |
| 242 | new rtc::RefCountedObject<cricket::EncoderStreamFactory>( |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 243 | cricket_codec, kDefaultMaxQp, screenshare, screenshare); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 244 | } else { |
| 245 | encoder_config.video_stream_factory = |
| 246 | new rtc::RefCountedObject<DefaultVideoStreamFactory>(); |
| 247 | } |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 248 | |
| 249 | // TODO(srte): Base this on encoder capabilities. |
| 250 | encoder_config.max_bitrate_bps = |
| 251 | config.encoder.max_data_rate.value_or(DataRate::kbps(10000)).bps(); |
| 252 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 253 | encoder_config.encoder_specific_settings = |
| 254 | CreateEncoderSpecificSettings(config); |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 255 | if (config.encoder.max_framerate) { |
| 256 | for (auto& layer : encoder_config.simulcast_layers) { |
| 257 | layer.max_framerate = *config.encoder.max_framerate; |
| 258 | } |
| 259 | } |
| 260 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 261 | return encoder_config; |
| 262 | } |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 263 | |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 264 | std::unique_ptr<FrameGenerator> CreateImageSlideGenerator( |
| 265 | Clock* clock, |
| 266 | VideoStreamConfig::Source::Slides slides, |
| 267 | int framerate) { |
| 268 | std::vector<std::string> paths = slides.images.paths; |
| 269 | for (std::string& path : paths) |
| 270 | path = TransformFilePath(path); |
| 271 | if (slides.images.crop.width || slides.images.crop.height) { |
| 272 | TimeDelta pause_duration = |
| 273 | slides.change_interval - slides.images.crop.scroll_duration; |
Sebastian Jansson | 5a00016 | 2019-04-12 09:21:32 | [diff] [blame] | 274 | RTC_CHECK_GE(pause_duration, TimeDelta::Zero()); |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 275 | int crop_width = slides.images.crop.width.value_or(slides.images.width); |
| 276 | int crop_height = slides.images.crop.height.value_or(slides.images.height); |
| 277 | RTC_CHECK_LE(crop_width, slides.images.width); |
| 278 | RTC_CHECK_LE(crop_height, slides.images.height); |
| 279 | return FrameGenerator::CreateScrollingInputFromYuvFiles( |
| 280 | clock, paths, slides.images.width, slides.images.height, crop_width, |
| 281 | crop_height, slides.images.crop.scroll_duration.ms(), |
| 282 | pause_duration.ms()); |
| 283 | } else { |
| 284 | return FrameGenerator::CreateFromYuvFile( |
| 285 | paths, slides.images.width, slides.images.height, |
| 286 | slides.change_interval.seconds<double>() * framerate); |
| 287 | } |
| 288 | } |
| 289 | |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 290 | std::unique_ptr<FrameGenerator> CreateFrameGenerator( |
| 291 | Clock* clock, |
| 292 | VideoStreamConfig::Source source) { |
| 293 | using Capture = VideoStreamConfig::Source::Capture; |
| 294 | switch (source.capture) { |
| 295 | case Capture::kGenerator: |
| 296 | return FrameGenerator::CreateSquareGenerator( |
| 297 | source.generator.width, source.generator.height, |
| 298 | source.generator.pixel_format, /*num_squares*/ absl::nullopt); |
| 299 | case Capture::kVideoFile: |
| 300 | RTC_CHECK(source.video_file.width && source.video_file.height); |
| 301 | return FrameGenerator::CreateFromYuvFile( |
Sebastian Jansson | d37307c | 2019-02-22 16:07:49 | [diff] [blame] | 302 | {TransformFilePath(source.video_file.name)}, source.video_file.width, |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 303 | source.video_file.height, /*frame_repeat_count*/ 1); |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 304 | case Capture::kGenerateSlides: |
| 305 | return FrameGenerator::CreateSlideGenerator( |
| 306 | source.slides.generator.width, source.slides.generator.height, |
| 307 | source.slides.change_interval.seconds<double>() * source.framerate); |
| 308 | case Capture::kImageSlides: |
| 309 | return CreateImageSlideGenerator(clock, source.slides, source.framerate); |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 310 | } |
| 311 | } |
| 312 | |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 313 | VideoReceiveStream::Config CreateVideoReceiveStreamConfig( |
| 314 | VideoStreamConfig config, |
| 315 | Transport* feedback_transport, |
| 316 | VideoReceiveStream::Decoder decoder, |
| 317 | rtc::VideoSinkInterface<VideoFrame>* renderer, |
| 318 | uint32_t local_ssrc, |
| 319 | uint32_t ssrc, |
| 320 | uint32_t rtx_ssrc) { |
| 321 | VideoReceiveStream::Config recv(feedback_transport); |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 322 | recv.rtp.transport_cc = config.stream.packet_feedback; |
| 323 | recv.rtp.local_ssrc = local_ssrc; |
| 324 | recv.rtp.extensions = GetVideoRtpExtensions(config); |
Sebastian Jansson | 24cf260 | 2019-06-07 13:24:57 | [diff] [blame] | 325 | |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 326 | RTC_DCHECK(!config.stream.use_rtx || |
| 327 | config.stream.nack_history_time > TimeDelta::Zero()); |
| 328 | recv.rtp.nack.rtp_history_ms = config.stream.nack_history_time.ms(); |
| 329 | recv.rtp.protected_by_flexfec = config.stream.use_flexfec; |
| 330 | recv.rtp.remote_ssrc = ssrc; |
| 331 | recv.decoders.push_back(decoder); |
| 332 | recv.renderer = renderer; |
| 333 | if (config.stream.use_rtx) { |
| 334 | recv.rtp.rtx_ssrc = rtx_ssrc; |
| 335 | recv.rtp.rtx_associated_payload_types[CallTest::kSendRtxPayloadType] = |
| 336 | CodecTypeToPayloadType(config.encoder.codec); |
| 337 | } |
| 338 | if (config.stream.use_ulpfec) { |
| 339 | recv.rtp.red_payload_type = CallTest::kRedPayloadType; |
| 340 | recv.rtp.ulpfec_payload_type = CallTest::kUlpfecPayloadType; |
| 341 | recv.rtp.rtx_associated_payload_types[CallTest::kRtxRedPayloadType] = |
| 342 | CallTest::kRedPayloadType; |
| 343 | } |
Sebastian Jansson | cc5be54 | 2019-04-01 11:34:55 | [diff] [blame] | 344 | recv.sync_group = config.render.sync_group; |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 345 | return recv; |
| 346 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 347 | } // namespace |
| 348 | |
| 349 | SendVideoStream::SendVideoStream(CallClient* sender, |
| 350 | VideoStreamConfig config, |
Sebastian Jansson | 9a4f38e | 2018-12-19 12:14:41 | [diff] [blame] | 351 | Transport* send_transport, |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 352 | VideoFrameMatcher* matcher) |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 353 | : sender_(sender), config_(config) { |
Mirko Bonadei | 317a1f0 | 2019-09-17 15:06:18 | [diff] [blame] | 354 | video_capturer_ = std::make_unique<FrameGeneratorCapturer>( |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 355 | sender_->clock_, CreateFrameGenerator(sender_->clock_, config.source), |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 356 | config.source.framerate, |
| 357 | *sender->time_controller_->GetTaskQueueFactory()); |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 358 | video_capturer_->Init(); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 359 | |
| 360 | using Encoder = VideoStreamConfig::Encoder; |
| 361 | using Codec = VideoStreamConfig::Encoder::Codec; |
| 362 | switch (config.encoder.implementation) { |
| 363 | case Encoder::Implementation::kFake: |
Jonas Olsson | a4d8737 | 2019-07-05 17:08:33 | [diff] [blame] | 364 | encoder_factory_ = |
Mirko Bonadei | 317a1f0 | 2019-09-17 15:06:18 | [diff] [blame] | 365 | std::make_unique<FunctionVideoEncoderFactory>([this]() { |
Jonas Olsson | a4d8737 | 2019-07-05 17:08:33 | [diff] [blame] | 366 | rtc::CritScope cs(&crit_); |
| 367 | std::unique_ptr<FakeEncoder> encoder; |
| 368 | if (config_.encoder.codec == Codec::kVideoCodecVP8) { |
Mirko Bonadei | 317a1f0 | 2019-09-17 15:06:18 | [diff] [blame] | 369 | encoder = std::make_unique<test::FakeVP8Encoder>(sender_->clock_); |
Jonas Olsson | a4d8737 | 2019-07-05 17:08:33 | [diff] [blame] | 370 | } else if (config_.encoder.codec == Codec::kVideoCodecGeneric) { |
Mirko Bonadei | 317a1f0 | 2019-09-17 15:06:18 | [diff] [blame] | 371 | encoder = std::make_unique<test::FakeEncoder>(sender_->clock_); |
Jonas Olsson | a4d8737 | 2019-07-05 17:08:33 | [diff] [blame] | 372 | } else { |
| 373 | RTC_NOTREACHED(); |
| 374 | } |
| 375 | fake_encoders_.push_back(encoder.get()); |
| 376 | if (config_.encoder.fake.max_rate.IsFinite()) |
| 377 | encoder->SetMaxBitrate(config_.encoder.fake.max_rate.kbps()); |
| 378 | return encoder; |
| 379 | }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 380 | break; |
| 381 | case VideoStreamConfig::Encoder::Implementation::kSoftware: |
| 382 | encoder_factory_.reset(new InternalEncoderFactory()); |
| 383 | break; |
| 384 | case VideoStreamConfig::Encoder::Implementation::kHardware: |
| 385 | encoder_factory_ = CreateHardwareEncoderFactory(); |
| 386 | break; |
| 387 | } |
| 388 | RTC_CHECK(encoder_factory_); |
| 389 | |
Jiawei Ou | c2ebe21 | 2018-11-08 18:02:56 | [diff] [blame] | 390 | bitrate_allocator_factory_ = CreateBuiltinVideoBitrateAllocatorFactory(); |
| 391 | RTC_CHECK(bitrate_allocator_factory_); |
| 392 | |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 393 | VideoEncoderConfig encoder_config = CreateVideoEncoderConfig(config); |
| 394 | for (size_t i = 0; i < encoder_config.number_of_streams; ++i) { |
| 395 | ssrcs_.push_back(sender->GetNextVideoSsrc()); |
| 396 | rtx_ssrcs_.push_back(sender->GetNextRtxSsrc()); |
| 397 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 398 | VideoSendStream::Config send_config = |
| 399 | CreateVideoSendStreamConfig(config, ssrcs_, send_transport); |
| 400 | send_config.encoder_settings.encoder_factory = encoder_factory_.get(); |
Jiawei Ou | c2ebe21 | 2018-11-08 18:02:56 | [diff] [blame] | 401 | send_config.encoder_settings.bitrate_allocator_factory = |
| 402 | bitrate_allocator_factory_.get(); |
| 403 | |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 404 | sender_->SendTask([&] { |
Sebastian Jansson | cc5be54 | 2019-04-01 11:34:55 | [diff] [blame] | 405 | if (config.stream.fec_controller_factory) { |
| 406 | send_stream_ = sender_->call_->CreateVideoSendStream( |
| 407 | std::move(send_config), std::move(encoder_config), |
| 408 | config.stream.fec_controller_factory->CreateFecController()); |
| 409 | } else { |
| 410 | send_stream_ = sender_->call_->CreateVideoSendStream( |
| 411 | std::move(send_config), std::move(encoder_config)); |
| 412 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 413 | |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 414 | if (matcher->Active()) { |
Mirko Bonadei | 317a1f0 | 2019-09-17 15:06:18 | [diff] [blame] | 415 | frame_tap_ = std::make_unique<ForwardingCapturedFrameTap>( |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 416 | sender_->clock_, matcher, video_capturer_.get()); |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 417 | send_stream_->SetSource(frame_tap_.get(), |
| 418 | config.encoder.degradation_preference); |
| 419 | } else { |
| 420 | send_stream_->SetSource(video_capturer_.get(), |
| 421 | config.encoder.degradation_preference); |
| 422 | } |
| 423 | }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 424 | } |
| 425 | |
| 426 | SendVideoStream::~SendVideoStream() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 427 | sender_->SendTask( |
| 428 | [this] { sender_->call_->DestroyVideoSendStream(send_stream_); }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 429 | } |
| 430 | |
| 431 | void SendVideoStream::Start() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 432 | sender_->SendTask([this] { |
| 433 | send_stream_->Start(); |
| 434 | sender_->call_->SignalChannelNetworkState(MediaType::VIDEO, kNetworkUp); |
| 435 | }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 436 | } |
| 437 | |
Christoffer Rodbro | 5f6abcf | 2019-02-08 10:00:10 | [diff] [blame] | 438 | void SendVideoStream::Stop() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 439 | sender_->SendTask([this] { send_stream_->Stop(); }); |
Christoffer Rodbro | 5f6abcf | 2019-02-08 10:00:10 | [diff] [blame] | 440 | } |
| 441 | |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 442 | void SendVideoStream::UpdateConfig( |
| 443 | std::function<void(VideoStreamConfig*)> modifier) { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 444 | sender_->SendTask([&] { |
| 445 | rtc::CritScope cs(&crit_); |
| 446 | VideoStreamConfig prior_config = config_; |
| 447 | modifier(&config_); |
| 448 | if (prior_config.encoder.fake.max_rate != config_.encoder.fake.max_rate) { |
| 449 | for (auto* encoder : fake_encoders_) { |
| 450 | encoder->SetMaxBitrate(config_.encoder.fake.max_rate.kbps()); |
| 451 | } |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 452 | } |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 453 | // TODO(srte): Add more conditions that should cause reconfiguration. |
| 454 | if (prior_config.encoder.max_framerate != config_.encoder.max_framerate) { |
| 455 | VideoEncoderConfig encoder_config = CreateVideoEncoderConfig(config_); |
| 456 | send_stream_->ReconfigureVideoEncoder(std::move(encoder_config)); |
| 457 | } |
| 458 | if (prior_config.source.framerate != config_.source.framerate) { |
| 459 | SetCaptureFramerate(config_.source.framerate); |
| 460 | } |
| 461 | }); |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 462 | } |
| 463 | |
Sebastian Jansson | e822668 | 2019-04-30 12:29:09 | [diff] [blame] | 464 | void SendVideoStream::UpdateActiveLayers(std::vector<bool> active_layers) { |
| 465 | sender_->task_queue_.PostTask([=] { |
| 466 | rtc::CritScope cs(&crit_); |
| 467 | if (config_.encoder.codec == |
| 468 | VideoStreamConfig::Encoder::Codec::kVideoCodecVP8) { |
| 469 | send_stream_->UpdateActiveSimulcastLayers(active_layers); |
| 470 | } else { |
| 471 | VideoEncoderConfig encoder_config = CreateVideoEncoderConfig(config_); |
| 472 | RTC_CHECK_EQ(encoder_config.simulcast_layers.size(), |
| 473 | active_layers.size()); |
| 474 | for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i) |
| 475 | encoder_config.simulcast_layers[i].active = active_layers[i]; |
| 476 | send_stream_->ReconfigureVideoEncoder(std::move(encoder_config)); |
| 477 | } |
| 478 | }); |
| 479 | } |
| 480 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 481 | void SendVideoStream::SetCaptureFramerate(int framerate) { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 482 | sender_->SendTask([&] { video_capturer_->ChangeFramerate(framerate); }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 483 | } |
| 484 | |
| 485 | VideoSendStream::Stats SendVideoStream::GetStats() const { |
| 486 | return send_stream_->GetStats(); |
| 487 | } |
| 488 | |
| 489 | ColumnPrinter SendVideoStream::StatsPrinter() { |
| 490 | return ColumnPrinter::Lambda( |
| 491 | "video_target_rate video_sent_rate width height", |
| 492 | [this](rtc::SimpleStringBuilder& sb) { |
| 493 | VideoSendStream::Stats video_stats = send_stream_->GetStats(); |
| 494 | int width = 0; |
| 495 | int height = 0; |
Mirko Bonadei | 739baf0 | 2019-01-27 16:29:42 | [diff] [blame] | 496 | for (const auto& stream_stat : video_stats.substreams) { |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 497 | width = std::max(width, stream_stat.second.width); |
| 498 | height = std::max(height, stream_stat.second.height); |
| 499 | } |
| 500 | sb.AppendFormat("%.0lf %.0lf %i %i", |
| 501 | video_stats.target_media_bitrate_bps / 8.0, |
| 502 | video_stats.media_bitrate_bps / 8.0, width, height); |
| 503 | }, |
| 504 | 64); |
| 505 | } |
| 506 | |
| 507 | ReceiveVideoStream::ReceiveVideoStream(CallClient* receiver, |
| 508 | VideoStreamConfig config, |
| 509 | SendVideoStream* send_stream, |
| 510 | size_t chosen_stream, |
Sebastian Jansson | 9a4f38e | 2018-12-19 12:14:41 | [diff] [blame] | 511 | Transport* feedback_transport, |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 512 | VideoFrameMatcher* matcher) |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 513 | : receiver_(receiver), config_(config) { |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 514 | if (config.encoder.codec == |
| 515 | VideoStreamConfig::Encoder::Codec::kVideoCodecGeneric) { |
Mirko Bonadei | 317a1f0 | 2019-09-17 15:06:18 | [diff] [blame] | 516 | decoder_factory_ = std::make_unique<FunctionVideoDecoderFactory>( |
| 517 | []() { return std::make_unique<FakeDecoder>(); }); |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 518 | } else { |
Mirko Bonadei | 317a1f0 | 2019-09-17 15:06:18 | [diff] [blame] | 519 | decoder_factory_ = std::make_unique<InternalDecoderFactory>(); |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 520 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 521 | |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 522 | VideoReceiveStream::Decoder decoder = |
| 523 | CreateMatchingDecoder(CodecTypeToPayloadType(config.encoder.codec), |
| 524 | CodecTypeToPayloadString(config.encoder.codec)); |
| 525 | decoder.decoder_factory = decoder_factory_.get(); |
| 526 | size_t num_streams = 1; |
| 527 | if (config.encoder.codec == VideoStreamConfig::Encoder::Codec::kVideoCodecVP8) |
| 528 | num_streams = config.encoder.layers.spatial; |
| 529 | for (size_t i = 0; i < num_streams; ++i) { |
| 530 | rtc::VideoSinkInterface<VideoFrame>* renderer = &fake_renderer_; |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 531 | if (matcher->Active()) { |
Sebastian Jansson | e9cac4f | 2019-06-24 15:10:55 | [diff] [blame] | 532 | render_taps_.emplace_back( |
Mirko Bonadei | 317a1f0 | 2019-09-17 15:06:18 | [diff] [blame] | 533 | std::make_unique<DecodedFrameTap>(receiver_->clock_, matcher, i)); |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 534 | renderer = render_taps_.back().get(); |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 535 | } |
| 536 | auto recv_config = CreateVideoReceiveStreamConfig( |
| 537 | config, feedback_transport, decoder, renderer, |
| 538 | receiver_->GetNextVideoLocalSsrc(), send_stream->ssrcs_[i], |
| 539 | send_stream->rtx_ssrcs_[i]); |
| 540 | if (config.stream.use_flexfec) { |
| 541 | RTC_DCHECK(num_streams == 1); |
| 542 | FlexfecReceiveStream::Config flexfec(feedback_transport); |
| 543 | flexfec.payload_type = CallTest::kFlexfecPayloadType; |
| 544 | flexfec.remote_ssrc = CallTest::kFlexfecSendSsrc; |
| 545 | flexfec.protected_media_ssrcs = send_stream->rtx_ssrcs_; |
| 546 | flexfec.local_ssrc = recv_config.rtp.local_ssrc; |
| 547 | receiver_->ssrc_media_types_[flexfec.remote_ssrc] = MediaType::VIDEO; |
Sebastian Jansson | 342f98b | 2019-06-18 14:08:23 | [diff] [blame] | 548 | |
| 549 | receiver_->SendTask([this, &flexfec] { |
| 550 | flecfec_stream_ = receiver_->call_->CreateFlexfecReceiveStream(flexfec); |
| 551 | }); |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 552 | } |
| 553 | receiver_->ssrc_media_types_[recv_config.rtp.remote_ssrc] = |
| 554 | MediaType::VIDEO; |
| 555 | if (config.stream.use_rtx) |
| 556 | receiver_->ssrc_media_types_[recv_config.rtp.rtx_ssrc] = MediaType::VIDEO; |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 557 | receiver_->SendTask([this, &recv_config] { |
| 558 | receive_streams_.push_back( |
| 559 | receiver_->call_->CreateVideoReceiveStream(std::move(recv_config))); |
| 560 | }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 561 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 562 | } |
| 563 | |
| 564 | ReceiveVideoStream::~ReceiveVideoStream() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 565 | receiver_->SendTask([this] { |
| 566 | for (auto* recv_stream : receive_streams_) |
| 567 | receiver_->call_->DestroyVideoReceiveStream(recv_stream); |
| 568 | if (flecfec_stream_) |
| 569 | receiver_->call_->DestroyFlexfecReceiveStream(flecfec_stream_); |
| 570 | }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 571 | } |
| 572 | |
Sebastian Jansson | 49a7843 | 2018-11-20 15:15:29 | [diff] [blame] | 573 | void ReceiveVideoStream::Start() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 574 | receiver_->SendTask([this] { |
| 575 | for (auto* recv_stream : receive_streams_) |
| 576 | recv_stream->Start(); |
| 577 | receiver_->call_->SignalChannelNetworkState(MediaType::VIDEO, kNetworkUp); |
| 578 | }); |
Sebastian Jansson | 49a7843 | 2018-11-20 15:15:29 | [diff] [blame] | 579 | } |
| 580 | |
Sebastian Jansson | bdfadd6 | 2019-02-08 12:34:57 | [diff] [blame] | 581 | void ReceiveVideoStream::Stop() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 582 | receiver_->SendTask([this] { |
| 583 | for (auto* recv_stream : receive_streams_) |
| 584 | recv_stream->Stop(); |
| 585 | }); |
Sebastian Jansson | bdfadd6 | 2019-02-08 12:34:57 | [diff] [blame] | 586 | } |
| 587 | |
Sebastian Jansson | f4481c8 | 2019-04-09 10:48:34 | [diff] [blame] | 588 | VideoReceiveStream::Stats ReceiveVideoStream::GetStats() const { |
| 589 | if (receive_streams_.empty()) |
| 590 | return VideoReceiveStream::Stats(); |
| 591 | // TODO(srte): Handle multiple receive streams. |
| 592 | return receive_streams_.front()->GetStats(); |
| 593 | } |
| 594 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 595 | VideoStreamPair::~VideoStreamPair() = default; |
| 596 | |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 597 | VideoStreamPair::VideoStreamPair(CallClient* sender, |
| 598 | CallClient* receiver, |
| 599 | VideoStreamConfig config) |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 600 | : config_(config), |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 601 | matcher_(config.hooks.frame_pair_handlers), |
| 602 | send_stream_(sender, config, sender->transport_.get(), &matcher_), |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 603 | receive_stream_(receiver, |
| 604 | config, |
| 605 | &send_stream_, |
| 606 | /*chosen_stream=*/0, |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 607 | receiver->transport_.get(), |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 608 | &matcher_) {} |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 609 | |
| 610 | } // namespace test |
| 611 | } // namespace webrtc |