Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | #include "test/scenario/video_stream.h" |
| 11 | |
| 12 | #include <algorithm> |
| 13 | #include <utility> |
| 14 | |
Sebastian Jansson | 9a4f38e | 2018-12-19 12:14:41 | [diff] [blame] | 15 | #include "absl/memory/memory.h" |
Danil Chapovalov | 99b71df | 2018-10-26 13:57:48 | [diff] [blame] | 16 | #include "api/test/video/function_video_encoder_factory.h" |
Jiawei Ou | c2ebe21 | 2018-11-08 18:02:56 | [diff] [blame] | 17 | #include "api/video/builtin_video_bitrate_allocator_factory.h" |
Steve Anton | 10542f2 | 2019-01-11 17:11:00 | [diff] [blame] | 18 | #include "media/base/media_constants.h" |
| 19 | #include "media/engine/internal_decoder_factory.h" |
| 20 | #include "media/engine/internal_encoder_factory.h" |
| 21 | #include "media/engine/webrtc_video_engine.h" |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 22 | #include "test/call_test.h" |
| 23 | #include "test/fake_encoder.h" |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 24 | #include "test/scenario/hardware_codecs.h" |
Steve Anton | 10542f2 | 2019-01-11 17:11:00 | [diff] [blame] | 25 | #include "test/testsupport/file_utils.h" |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 26 | |
| 27 | namespace webrtc { |
| 28 | namespace test { |
| 29 | namespace { |
Elad Alon | d8d3248 | 2019-02-18 22:45:57 | [diff] [blame] | 30 | enum : int { // The first valid value is 1. |
| 31 | kTransportSequenceNumberExtensionId = 1, |
| 32 | kVideoContentTypeExtensionId, |
| 33 | kVideoRotationRtpExtensionId, |
| 34 | }; |
| 35 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 36 | constexpr int kDefaultMaxQp = cricket::WebRtcVideoChannel::kDefaultQpMax; |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 37 | uint8_t CodecTypeToPayloadType(VideoCodecType codec_type) { |
| 38 | switch (codec_type) { |
| 39 | case VideoCodecType::kVideoCodecGeneric: |
| 40 | return CallTest::kFakeVideoSendPayloadType; |
| 41 | case VideoCodecType::kVideoCodecVP8: |
| 42 | return CallTest::kPayloadTypeVP8; |
| 43 | case VideoCodecType::kVideoCodecVP9: |
| 44 | return CallTest::kPayloadTypeVP9; |
| 45 | case VideoCodecType::kVideoCodecH264: |
| 46 | return CallTest::kPayloadTypeH264; |
| 47 | default: |
| 48 | RTC_NOTREACHED(); |
| 49 | } |
| 50 | return {}; |
| 51 | } |
| 52 | std::string CodecTypeToCodecName(VideoCodecType codec_type) { |
| 53 | switch (codec_type) { |
| 54 | case VideoCodecType::kVideoCodecGeneric: |
| 55 | return ""; |
| 56 | case VideoCodecType::kVideoCodecVP8: |
| 57 | return cricket::kVp8CodecName; |
| 58 | case VideoCodecType::kVideoCodecVP9: |
| 59 | return cricket::kVp9CodecName; |
| 60 | case VideoCodecType::kVideoCodecH264: |
| 61 | return cricket::kH264CodecName; |
| 62 | default: |
| 63 | RTC_NOTREACHED(); |
| 64 | } |
| 65 | return {}; |
| 66 | } |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 67 | VideoEncoderConfig::ContentType ConvertContentType( |
| 68 | VideoStreamConfig::Encoder::ContentType content_type) { |
| 69 | switch (content_type) { |
| 70 | case VideoStreamConfig::Encoder::ContentType::kVideo: |
| 71 | return VideoEncoderConfig::ContentType::kRealtimeVideo; |
| 72 | break; |
| 73 | case VideoStreamConfig::Encoder::ContentType::kScreen: |
| 74 | return VideoEncoderConfig::ContentType::kScreen; |
| 75 | } |
| 76 | } |
Sebastian Jansson | 06c5145 | 2019-02-18 15:55:15 | [diff] [blame] | 77 | InterLayerPredMode ToInterLayerPredMode( |
| 78 | VideoStreamConfig::Encoder::Layers::Prediction value) { |
| 79 | using Pred = VideoStreamConfig::Encoder::Layers::Prediction; |
| 80 | switch (value) { |
| 81 | case Pred::kTemporalOnly: |
| 82 | return InterLayerPredMode::kOff; |
| 83 | case Pred::kSpatialOnKey: |
| 84 | return InterLayerPredMode::kOnKeyPic; |
| 85 | case Pred::kFull: |
| 86 | return InterLayerPredMode::kOn; |
| 87 | } |
| 88 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 89 | std::vector<RtpExtension> GetVideoRtpExtensions( |
| 90 | const VideoStreamConfig config) { |
Sebastian Jansson | 24cf260 | 2019-06-07 13:24:57 | [diff] [blame^] | 91 | std::vector<RtpExtension> res = { |
| 92 | RtpExtension(RtpExtension::kVideoContentTypeUri, |
| 93 | kVideoContentTypeExtensionId), |
| 94 | RtpExtension(RtpExtension::kVideoRotationUri, |
| 95 | kVideoRotationRtpExtensionId)}; |
| 96 | if (config.stream.packet_feedback) { |
| 97 | res.push_back(RtpExtension(RtpExtension::kTransportSequenceNumberUri, |
| 98 | kTransportSequenceNumberExtensionId)); |
| 99 | } |
| 100 | return res; |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 101 | } |
| 102 | |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 103 | std::string TransformFilePath(std::string path) { |
| 104 | static const std::string resource_prefix = "res://"; |
| 105 | int ext_pos = path.rfind("."); |
| 106 | if (ext_pos < 0) { |
| 107 | return test::ResourcePath(path, "yuv"); |
| 108 | } else if (path.find(resource_prefix) == 0) { |
| 109 | std::string name = path.substr(resource_prefix.length(), ext_pos); |
| 110 | std::string ext = path.substr(ext_pos, path.size()); |
| 111 | return test::ResourcePath(name, ext); |
| 112 | } |
| 113 | return path; |
| 114 | } |
| 115 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 116 | VideoSendStream::Config CreateVideoSendStreamConfig(VideoStreamConfig config, |
| 117 | std::vector<uint32_t> ssrcs, |
| 118 | Transport* send_transport) { |
| 119 | VideoSendStream::Config send_config(send_transport); |
| 120 | send_config.rtp.payload_name = CodecTypeToPayloadString(config.encoder.codec); |
| 121 | send_config.rtp.payload_type = CodecTypeToPayloadType(config.encoder.codec); |
| 122 | |
| 123 | send_config.rtp.ssrcs = ssrcs; |
| 124 | send_config.rtp.extensions = GetVideoRtpExtensions(config); |
| 125 | |
| 126 | if (config.stream.use_flexfec) { |
| 127 | send_config.rtp.flexfec.payload_type = CallTest::kFlexfecPayloadType; |
| 128 | send_config.rtp.flexfec.ssrc = CallTest::kFlexfecSendSsrc; |
| 129 | send_config.rtp.flexfec.protected_media_ssrcs = ssrcs; |
| 130 | } |
| 131 | if (config.stream.use_ulpfec) { |
| 132 | send_config.rtp.ulpfec.red_payload_type = CallTest::kRedPayloadType; |
| 133 | send_config.rtp.ulpfec.ulpfec_payload_type = CallTest::kUlpfecPayloadType; |
| 134 | send_config.rtp.ulpfec.red_rtx_payload_type = CallTest::kRtxRedPayloadType; |
| 135 | } |
| 136 | return send_config; |
| 137 | } |
| 138 | rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings> |
Sebastian Jansson | 06c5145 | 2019-02-18 15:55:15 | [diff] [blame] | 139 | CreateVp9SpecificSettings(VideoStreamConfig video_config) { |
| 140 | constexpr auto kScreen = VideoStreamConfig::Encoder::ContentType::kScreen; |
| 141 | VideoStreamConfig::Encoder conf = video_config.encoder; |
| 142 | VideoCodecVP9 vp9 = VideoEncoder::GetDefaultVp9Settings(); |
| 143 | vp9.frameDroppingOn = conf.frame_dropping; |
| 144 | vp9.keyFrameInterval = conf.key_frame_interval.value_or(0); |
| 145 | vp9.numberOfTemporalLayers = static_cast<uint8_t>(conf.layers.temporal); |
| 146 | vp9.numberOfSpatialLayers = static_cast<uint8_t>(conf.layers.spatial); |
| 147 | vp9.interLayerPred = ToInterLayerPredMode(conf.layers.prediction); |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 148 | |
Sebastian Jansson | 06c5145 | 2019-02-18 15:55:15 | [diff] [blame] | 149 | if (conf.content_type == kScreen && |
| 150 | (video_config.source.framerate > 5 || conf.layers.spatial >= 3)) { |
| 151 | vp9.flexibleMode = true; |
| 152 | } |
| 153 | |
| 154 | if (conf.content_type == kScreen || |
| 155 | conf.layers.temporal * conf.layers.spatial) { |
| 156 | vp9.automaticResizeOn = false; |
| 157 | vp9.denoisingOn = false; |
| 158 | } else { |
| 159 | vp9.automaticResizeOn = conf.single.automatic_scaling; |
| 160 | vp9.denoisingOn = conf.single.denoising; |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 161 | } |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 162 | return new rtc::RefCountedObject< |
Sebastian Jansson | 06c5145 | 2019-02-18 15:55:15 | [diff] [blame] | 163 | VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9); |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 164 | } |
| 165 | |
| 166 | rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings> |
| 167 | CreateVp8SpecificSettings(VideoStreamConfig config) { |
| 168 | VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings(); |
| 169 | vp8_settings.frameDroppingOn = config.encoder.frame_dropping; |
| 170 | vp8_settings.keyFrameInterval = config.encoder.key_frame_interval.value_or(0); |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 171 | vp8_settings.numberOfTemporalLayers = config.encoder.layers.temporal; |
| 172 | if (config.encoder.layers.spatial * config.encoder.layers.temporal > 1) { |
| 173 | vp8_settings.automaticResizeOn = false; |
| 174 | vp8_settings.denoisingOn = false; |
| 175 | } else { |
| 176 | vp8_settings.automaticResizeOn = config.encoder.single.automatic_scaling; |
| 177 | vp8_settings.denoisingOn = config.encoder.single.denoising; |
| 178 | } |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 179 | return new rtc::RefCountedObject< |
| 180 | VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings); |
| 181 | } |
| 182 | |
| 183 | rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings> |
| 184 | CreateH264SpecificSettings(VideoStreamConfig config) { |
Sebastian Jansson | 06c5145 | 2019-02-18 15:55:15 | [diff] [blame] | 185 | RTC_DCHECK_EQ(config.encoder.layers.temporal, 1); |
| 186 | RTC_DCHECK_EQ(config.encoder.layers.spatial, 1); |
| 187 | |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 188 | VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings(); |
| 189 | h264_settings.frameDroppingOn = config.encoder.frame_dropping; |
| 190 | h264_settings.keyFrameInterval = |
| 191 | config.encoder.key_frame_interval.value_or(0); |
| 192 | return new rtc::RefCountedObject< |
| 193 | VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings); |
| 194 | } |
| 195 | |
| 196 | rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings> |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 197 | CreateEncoderSpecificSettings(VideoStreamConfig config) { |
| 198 | using Codec = VideoStreamConfig::Encoder::Codec; |
| 199 | switch (config.encoder.codec) { |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 200 | case Codec::kVideoCodecH264: |
| 201 | return CreateH264SpecificSettings(config); |
| 202 | case Codec::kVideoCodecVP8: |
| 203 | return CreateVp8SpecificSettings(config); |
| 204 | case Codec::kVideoCodecVP9: |
| 205 | return CreateVp9SpecificSettings(config); |
| 206 | case Codec::kVideoCodecGeneric: |
| 207 | return nullptr; |
| 208 | case Codec::kVideoCodecMultiplex: |
| 209 | RTC_NOTREACHED(); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 210 | return nullptr; |
| 211 | } |
| 212 | } |
| 213 | |
| 214 | VideoEncoderConfig CreateVideoEncoderConfig(VideoStreamConfig config) { |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 215 | VideoEncoderConfig encoder_config; |
| 216 | encoder_config.codec_type = config.encoder.codec; |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 217 | encoder_config.content_type = ConvertContentType(config.encoder.content_type); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 218 | encoder_config.video_format = |
| 219 | SdpVideoFormat(CodecTypeToPayloadString(config.encoder.codec), {}); |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 220 | |
Sebastian Jansson | e822668 | 2019-04-30 12:29:09 | [diff] [blame] | 221 | encoder_config.number_of_streams = 1; |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 222 | if (config.encoder.codec == VideoStreamConfig::Encoder::Codec::kVideoCodecVP8) |
Sebastian Jansson | e822668 | 2019-04-30 12:29:09 | [diff] [blame] | 223 | encoder_config.number_of_streams = |
| 224 | static_cast<size_t>(config.encoder.layers.spatial); |
| 225 | encoder_config.simulcast_layers = |
| 226 | std::vector<VideoStream>(config.encoder.layers.spatial); |
Sebastian Jansson | 985ee68 | 2018-11-12 15:33:29 | [diff] [blame] | 227 | encoder_config.min_transmit_bitrate_bps = config.stream.pad_to_rate.bps(); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 228 | |
| 229 | std::string cricket_codec = CodecTypeToCodecName(config.encoder.codec); |
| 230 | if (!cricket_codec.empty()) { |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 231 | bool screenshare = config.encoder.content_type == |
| 232 | VideoStreamConfig::Encoder::ContentType::kScreen; |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 233 | encoder_config.video_stream_factory = |
| 234 | new rtc::RefCountedObject<cricket::EncoderStreamFactory>( |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 235 | cricket_codec, kDefaultMaxQp, screenshare, screenshare); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 236 | } else { |
| 237 | encoder_config.video_stream_factory = |
| 238 | new rtc::RefCountedObject<DefaultVideoStreamFactory>(); |
| 239 | } |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 240 | |
| 241 | // TODO(srte): Base this on encoder capabilities. |
| 242 | encoder_config.max_bitrate_bps = |
| 243 | config.encoder.max_data_rate.value_or(DataRate::kbps(10000)).bps(); |
| 244 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 245 | encoder_config.encoder_specific_settings = |
| 246 | CreateEncoderSpecificSettings(config); |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 247 | if (config.encoder.max_framerate) { |
| 248 | for (auto& layer : encoder_config.simulcast_layers) { |
| 249 | layer.max_framerate = *config.encoder.max_framerate; |
| 250 | } |
| 251 | } |
| 252 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 253 | return encoder_config; |
| 254 | } |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 255 | |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 256 | std::unique_ptr<FrameGenerator> CreateImageSlideGenerator( |
| 257 | Clock* clock, |
| 258 | VideoStreamConfig::Source::Slides slides, |
| 259 | int framerate) { |
| 260 | std::vector<std::string> paths = slides.images.paths; |
| 261 | for (std::string& path : paths) |
| 262 | path = TransformFilePath(path); |
| 263 | if (slides.images.crop.width || slides.images.crop.height) { |
| 264 | TimeDelta pause_duration = |
| 265 | slides.change_interval - slides.images.crop.scroll_duration; |
Sebastian Jansson | 5a00016 | 2019-04-12 09:21:32 | [diff] [blame] | 266 | RTC_CHECK_GE(pause_duration, TimeDelta::Zero()); |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 267 | int crop_width = slides.images.crop.width.value_or(slides.images.width); |
| 268 | int crop_height = slides.images.crop.height.value_or(slides.images.height); |
| 269 | RTC_CHECK_LE(crop_width, slides.images.width); |
| 270 | RTC_CHECK_LE(crop_height, slides.images.height); |
| 271 | return FrameGenerator::CreateScrollingInputFromYuvFiles( |
| 272 | clock, paths, slides.images.width, slides.images.height, crop_width, |
| 273 | crop_height, slides.images.crop.scroll_duration.ms(), |
| 274 | pause_duration.ms()); |
| 275 | } else { |
| 276 | return FrameGenerator::CreateFromYuvFile( |
| 277 | paths, slides.images.width, slides.images.height, |
| 278 | slides.change_interval.seconds<double>() * framerate); |
| 279 | } |
| 280 | } |
| 281 | |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 282 | std::unique_ptr<FrameGenerator> CreateFrameGenerator( |
| 283 | Clock* clock, |
| 284 | VideoStreamConfig::Source source) { |
| 285 | using Capture = VideoStreamConfig::Source::Capture; |
| 286 | switch (source.capture) { |
| 287 | case Capture::kGenerator: |
| 288 | return FrameGenerator::CreateSquareGenerator( |
| 289 | source.generator.width, source.generator.height, |
| 290 | source.generator.pixel_format, /*num_squares*/ absl::nullopt); |
| 291 | case Capture::kVideoFile: |
| 292 | RTC_CHECK(source.video_file.width && source.video_file.height); |
| 293 | return FrameGenerator::CreateFromYuvFile( |
Sebastian Jansson | d37307c | 2019-02-22 16:07:49 | [diff] [blame] | 294 | {TransformFilePath(source.video_file.name)}, source.video_file.width, |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 295 | source.video_file.height, /*frame_repeat_count*/ 1); |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 296 | case Capture::kGenerateSlides: |
| 297 | return FrameGenerator::CreateSlideGenerator( |
| 298 | source.slides.generator.width, source.slides.generator.height, |
| 299 | source.slides.change_interval.seconds<double>() * source.framerate); |
| 300 | case Capture::kImageSlides: |
| 301 | return CreateImageSlideGenerator(clock, source.slides, source.framerate); |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 302 | } |
| 303 | } |
| 304 | |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 305 | VideoReceiveStream::Config CreateVideoReceiveStreamConfig( |
| 306 | VideoStreamConfig config, |
| 307 | Transport* feedback_transport, |
| 308 | VideoReceiveStream::Decoder decoder, |
| 309 | rtc::VideoSinkInterface<VideoFrame>* renderer, |
| 310 | uint32_t local_ssrc, |
| 311 | uint32_t ssrc, |
| 312 | uint32_t rtx_ssrc) { |
| 313 | VideoReceiveStream::Config recv(feedback_transport); |
| 314 | recv.rtp.remb = !config.stream.packet_feedback; |
| 315 | recv.rtp.transport_cc = config.stream.packet_feedback; |
| 316 | recv.rtp.local_ssrc = local_ssrc; |
| 317 | recv.rtp.extensions = GetVideoRtpExtensions(config); |
Sebastian Jansson | 24cf260 | 2019-06-07 13:24:57 | [diff] [blame^] | 318 | |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 319 | RTC_DCHECK(!config.stream.use_rtx || |
| 320 | config.stream.nack_history_time > TimeDelta::Zero()); |
| 321 | recv.rtp.nack.rtp_history_ms = config.stream.nack_history_time.ms(); |
| 322 | recv.rtp.protected_by_flexfec = config.stream.use_flexfec; |
| 323 | recv.rtp.remote_ssrc = ssrc; |
| 324 | recv.decoders.push_back(decoder); |
| 325 | recv.renderer = renderer; |
| 326 | if (config.stream.use_rtx) { |
| 327 | recv.rtp.rtx_ssrc = rtx_ssrc; |
| 328 | recv.rtp.rtx_associated_payload_types[CallTest::kSendRtxPayloadType] = |
| 329 | CodecTypeToPayloadType(config.encoder.codec); |
| 330 | } |
| 331 | if (config.stream.use_ulpfec) { |
| 332 | recv.rtp.red_payload_type = CallTest::kRedPayloadType; |
| 333 | recv.rtp.ulpfec_payload_type = CallTest::kUlpfecPayloadType; |
| 334 | recv.rtp.rtx_associated_payload_types[CallTest::kRtxRedPayloadType] = |
| 335 | CallTest::kRedPayloadType; |
| 336 | } |
Sebastian Jansson | cc5be54 | 2019-04-01 11:34:55 | [diff] [blame] | 337 | recv.sync_group = config.render.sync_group; |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 338 | return recv; |
| 339 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 340 | } // namespace |
| 341 | |
| 342 | SendVideoStream::SendVideoStream(CallClient* sender, |
| 343 | VideoStreamConfig config, |
Sebastian Jansson | 9a4f38e | 2018-12-19 12:14:41 | [diff] [blame] | 344 | Transport* send_transport, |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 345 | VideoFrameMatcher* matcher) |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 346 | : sender_(sender), config_(config) { |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 347 | video_capturer_ = absl::make_unique<FrameGeneratorCapturer>( |
| 348 | sender_->clock_, CreateFrameGenerator(sender_->clock_, config.source), |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 349 | config.source.framerate, |
| 350 | *sender->time_controller_->GetTaskQueueFactory()); |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 351 | video_capturer_->Init(); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 352 | |
| 353 | using Encoder = VideoStreamConfig::Encoder; |
| 354 | using Codec = VideoStreamConfig::Encoder::Codec; |
| 355 | switch (config.encoder.implementation) { |
| 356 | case Encoder::Implementation::kFake: |
| 357 | if (config.encoder.codec == Codec::kVideoCodecGeneric) { |
| 358 | encoder_factory_ = |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 359 | absl::make_unique<FunctionVideoEncoderFactory>([this]() { |
| 360 | rtc::CritScope cs(&crit_); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 361 | auto encoder = |
| 362 | absl::make_unique<test::FakeEncoder>(sender_->clock_); |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 363 | fake_encoders_.push_back(encoder.get()); |
| 364 | if (config_.encoder.fake.max_rate.IsFinite()) |
| 365 | encoder->SetMaxBitrate(config_.encoder.fake.max_rate.kbps()); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 366 | return encoder; |
| 367 | }); |
| 368 | } else { |
| 369 | RTC_NOTREACHED(); |
| 370 | } |
| 371 | break; |
| 372 | case VideoStreamConfig::Encoder::Implementation::kSoftware: |
| 373 | encoder_factory_.reset(new InternalEncoderFactory()); |
| 374 | break; |
| 375 | case VideoStreamConfig::Encoder::Implementation::kHardware: |
| 376 | encoder_factory_ = CreateHardwareEncoderFactory(); |
| 377 | break; |
| 378 | } |
| 379 | RTC_CHECK(encoder_factory_); |
| 380 | |
Jiawei Ou | c2ebe21 | 2018-11-08 18:02:56 | [diff] [blame] | 381 | bitrate_allocator_factory_ = CreateBuiltinVideoBitrateAllocatorFactory(); |
| 382 | RTC_CHECK(bitrate_allocator_factory_); |
| 383 | |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 384 | VideoEncoderConfig encoder_config = CreateVideoEncoderConfig(config); |
| 385 | for (size_t i = 0; i < encoder_config.number_of_streams; ++i) { |
| 386 | ssrcs_.push_back(sender->GetNextVideoSsrc()); |
| 387 | rtx_ssrcs_.push_back(sender->GetNextRtxSsrc()); |
| 388 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 389 | VideoSendStream::Config send_config = |
| 390 | CreateVideoSendStreamConfig(config, ssrcs_, send_transport); |
| 391 | send_config.encoder_settings.encoder_factory = encoder_factory_.get(); |
Jiawei Ou | c2ebe21 | 2018-11-08 18:02:56 | [diff] [blame] | 392 | send_config.encoder_settings.bitrate_allocator_factory = |
| 393 | bitrate_allocator_factory_.get(); |
| 394 | |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 395 | sender_->SendTask([&] { |
Sebastian Jansson | cc5be54 | 2019-04-01 11:34:55 | [diff] [blame] | 396 | if (config.stream.fec_controller_factory) { |
| 397 | send_stream_ = sender_->call_->CreateVideoSendStream( |
| 398 | std::move(send_config), std::move(encoder_config), |
| 399 | config.stream.fec_controller_factory->CreateFecController()); |
| 400 | } else { |
| 401 | send_stream_ = sender_->call_->CreateVideoSendStream( |
| 402 | std::move(send_config), std::move(encoder_config)); |
| 403 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 404 | |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 405 | if (matcher->Active()) { |
| 406 | frame_tap_ = absl::make_unique<ForwardingCapturedFrameTap>( |
| 407 | sender_->clock_, matcher, video_capturer_.get()); |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 408 | send_stream_->SetSource(frame_tap_.get(), |
| 409 | config.encoder.degradation_preference); |
| 410 | } else { |
| 411 | send_stream_->SetSource(video_capturer_.get(), |
| 412 | config.encoder.degradation_preference); |
| 413 | } |
| 414 | }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 415 | } |
| 416 | |
| 417 | SendVideoStream::~SendVideoStream() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 418 | sender_->SendTask( |
| 419 | [this] { sender_->call_->DestroyVideoSendStream(send_stream_); }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 420 | } |
| 421 | |
| 422 | void SendVideoStream::Start() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 423 | sender_->SendTask([this] { |
| 424 | send_stream_->Start(); |
| 425 | sender_->call_->SignalChannelNetworkState(MediaType::VIDEO, kNetworkUp); |
| 426 | }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 427 | } |
| 428 | |
Christoffer Rodbro | 5f6abcf | 2019-02-08 10:00:10 | [diff] [blame] | 429 | void SendVideoStream::Stop() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 430 | sender_->SendTask([this] { send_stream_->Stop(); }); |
Christoffer Rodbro | 5f6abcf | 2019-02-08 10:00:10 | [diff] [blame] | 431 | } |
| 432 | |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 433 | void SendVideoStream::UpdateConfig( |
| 434 | std::function<void(VideoStreamConfig*)> modifier) { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 435 | sender_->SendTask([&] { |
| 436 | rtc::CritScope cs(&crit_); |
| 437 | VideoStreamConfig prior_config = config_; |
| 438 | modifier(&config_); |
| 439 | if (prior_config.encoder.fake.max_rate != config_.encoder.fake.max_rate) { |
| 440 | for (auto* encoder : fake_encoders_) { |
| 441 | encoder->SetMaxBitrate(config_.encoder.fake.max_rate.kbps()); |
| 442 | } |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 443 | } |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 444 | // TODO(srte): Add more conditions that should cause reconfiguration. |
| 445 | if (prior_config.encoder.max_framerate != config_.encoder.max_framerate) { |
| 446 | VideoEncoderConfig encoder_config = CreateVideoEncoderConfig(config_); |
| 447 | send_stream_->ReconfigureVideoEncoder(std::move(encoder_config)); |
| 448 | } |
| 449 | if (prior_config.source.framerate != config_.source.framerate) { |
| 450 | SetCaptureFramerate(config_.source.framerate); |
| 451 | } |
| 452 | }); |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 453 | } |
| 454 | |
Sebastian Jansson | e822668 | 2019-04-30 12:29:09 | [diff] [blame] | 455 | void SendVideoStream::UpdateActiveLayers(std::vector<bool> active_layers) { |
| 456 | sender_->task_queue_.PostTask([=] { |
| 457 | rtc::CritScope cs(&crit_); |
| 458 | if (config_.encoder.codec == |
| 459 | VideoStreamConfig::Encoder::Codec::kVideoCodecVP8) { |
| 460 | send_stream_->UpdateActiveSimulcastLayers(active_layers); |
| 461 | } else { |
| 462 | VideoEncoderConfig encoder_config = CreateVideoEncoderConfig(config_); |
| 463 | RTC_CHECK_EQ(encoder_config.simulcast_layers.size(), |
| 464 | active_layers.size()); |
| 465 | for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i) |
| 466 | encoder_config.simulcast_layers[i].active = active_layers[i]; |
| 467 | send_stream_->ReconfigureVideoEncoder(std::move(encoder_config)); |
| 468 | } |
| 469 | }); |
| 470 | } |
| 471 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 472 | void SendVideoStream::SetCaptureFramerate(int framerate) { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 473 | sender_->SendTask([&] { video_capturer_->ChangeFramerate(framerate); }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 474 | } |
| 475 | |
| 476 | VideoSendStream::Stats SendVideoStream::GetStats() const { |
| 477 | return send_stream_->GetStats(); |
| 478 | } |
| 479 | |
| 480 | ColumnPrinter SendVideoStream::StatsPrinter() { |
| 481 | return ColumnPrinter::Lambda( |
| 482 | "video_target_rate video_sent_rate width height", |
| 483 | [this](rtc::SimpleStringBuilder& sb) { |
| 484 | VideoSendStream::Stats video_stats = send_stream_->GetStats(); |
| 485 | int width = 0; |
| 486 | int height = 0; |
Mirko Bonadei | 739baf0 | 2019-01-27 16:29:42 | [diff] [blame] | 487 | for (const auto& stream_stat : video_stats.substreams) { |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 488 | width = std::max(width, stream_stat.second.width); |
| 489 | height = std::max(height, stream_stat.second.height); |
| 490 | } |
| 491 | sb.AppendFormat("%.0lf %.0lf %i %i", |
| 492 | video_stats.target_media_bitrate_bps / 8.0, |
| 493 | video_stats.media_bitrate_bps / 8.0, width, height); |
| 494 | }, |
| 495 | 64); |
| 496 | } |
| 497 | |
| 498 | ReceiveVideoStream::ReceiveVideoStream(CallClient* receiver, |
| 499 | VideoStreamConfig config, |
| 500 | SendVideoStream* send_stream, |
| 501 | size_t chosen_stream, |
Sebastian Jansson | 9a4f38e | 2018-12-19 12:14:41 | [diff] [blame] | 502 | Transport* feedback_transport, |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 503 | VideoFrameMatcher* matcher) |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 504 | : receiver_(receiver), config_(config) { |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 505 | if (config.encoder.codec == |
| 506 | VideoStreamConfig::Encoder::Codec::kVideoCodecGeneric) { |
| 507 | decoder_factory_ = absl::make_unique<FunctionVideoDecoderFactory>( |
| 508 | []() { return absl::make_unique<FakeDecoder>(); }); |
| 509 | } else { |
| 510 | decoder_factory_ = absl::make_unique<InternalDecoderFactory>(); |
| 511 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 512 | |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 513 | VideoReceiveStream::Decoder decoder = |
| 514 | CreateMatchingDecoder(CodecTypeToPayloadType(config.encoder.codec), |
| 515 | CodecTypeToPayloadString(config.encoder.codec)); |
| 516 | decoder.decoder_factory = decoder_factory_.get(); |
| 517 | size_t num_streams = 1; |
| 518 | if (config.encoder.codec == VideoStreamConfig::Encoder::Codec::kVideoCodecVP8) |
| 519 | num_streams = config.encoder.layers.spatial; |
| 520 | for (size_t i = 0; i < num_streams; ++i) { |
| 521 | rtc::VideoSinkInterface<VideoFrame>* renderer = &fake_renderer_; |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 522 | if (matcher->Active()) { |
| 523 | render_taps_.emplace_back(absl::make_unique<DecodedFrameTap>(matcher, i)); |
| 524 | renderer = render_taps_.back().get(); |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 525 | } |
| 526 | auto recv_config = CreateVideoReceiveStreamConfig( |
| 527 | config, feedback_transport, decoder, renderer, |
| 528 | receiver_->GetNextVideoLocalSsrc(), send_stream->ssrcs_[i], |
| 529 | send_stream->rtx_ssrcs_[i]); |
| 530 | if (config.stream.use_flexfec) { |
| 531 | RTC_DCHECK(num_streams == 1); |
| 532 | FlexfecReceiveStream::Config flexfec(feedback_transport); |
| 533 | flexfec.payload_type = CallTest::kFlexfecPayloadType; |
| 534 | flexfec.remote_ssrc = CallTest::kFlexfecSendSsrc; |
| 535 | flexfec.protected_media_ssrcs = send_stream->rtx_ssrcs_; |
| 536 | flexfec.local_ssrc = recv_config.rtp.local_ssrc; |
| 537 | receiver_->ssrc_media_types_[flexfec.remote_ssrc] = MediaType::VIDEO; |
| 538 | flecfec_stream_ = receiver_->call_->CreateFlexfecReceiveStream(flexfec); |
| 539 | } |
| 540 | receiver_->ssrc_media_types_[recv_config.rtp.remote_ssrc] = |
| 541 | MediaType::VIDEO; |
| 542 | if (config.stream.use_rtx) |
| 543 | receiver_->ssrc_media_types_[recv_config.rtp.rtx_ssrc] = MediaType::VIDEO; |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 544 | receiver_->SendTask([this, &recv_config] { |
| 545 | receive_streams_.push_back( |
| 546 | receiver_->call_->CreateVideoReceiveStream(std::move(recv_config))); |
| 547 | }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 548 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 549 | } |
| 550 | |
| 551 | ReceiveVideoStream::~ReceiveVideoStream() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 552 | receiver_->SendTask([this] { |
| 553 | for (auto* recv_stream : receive_streams_) |
| 554 | receiver_->call_->DestroyVideoReceiveStream(recv_stream); |
| 555 | if (flecfec_stream_) |
| 556 | receiver_->call_->DestroyFlexfecReceiveStream(flecfec_stream_); |
| 557 | }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 558 | } |
| 559 | |
Sebastian Jansson | 49a7843 | 2018-11-20 15:15:29 | [diff] [blame] | 560 | void ReceiveVideoStream::Start() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 561 | receiver_->SendTask([this] { |
| 562 | for (auto* recv_stream : receive_streams_) |
| 563 | recv_stream->Start(); |
| 564 | receiver_->call_->SignalChannelNetworkState(MediaType::VIDEO, kNetworkUp); |
| 565 | }); |
Sebastian Jansson | 49a7843 | 2018-11-20 15:15:29 | [diff] [blame] | 566 | } |
| 567 | |
Sebastian Jansson | bdfadd6 | 2019-02-08 12:34:57 | [diff] [blame] | 568 | void ReceiveVideoStream::Stop() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 569 | receiver_->SendTask([this] { |
| 570 | for (auto* recv_stream : receive_streams_) |
| 571 | recv_stream->Stop(); |
| 572 | }); |
Sebastian Jansson | bdfadd6 | 2019-02-08 12:34:57 | [diff] [blame] | 573 | } |
| 574 | |
Sebastian Jansson | f4481c8 | 2019-04-09 10:48:34 | [diff] [blame] | 575 | VideoReceiveStream::Stats ReceiveVideoStream::GetStats() const { |
| 576 | if (receive_streams_.empty()) |
| 577 | return VideoReceiveStream::Stats(); |
| 578 | // TODO(srte): Handle multiple receive streams. |
| 579 | return receive_streams_.front()->GetStats(); |
| 580 | } |
| 581 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 582 | VideoStreamPair::~VideoStreamPair() = default; |
| 583 | |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 584 | VideoStreamPair::VideoStreamPair(CallClient* sender, |
| 585 | CallClient* receiver, |
| 586 | VideoStreamConfig config) |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 587 | : config_(config), |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 588 | matcher_(config.hooks.frame_pair_handlers), |
| 589 | send_stream_(sender, config, sender->transport_.get(), &matcher_), |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 590 | receive_stream_(receiver, |
| 591 | config, |
| 592 | &send_stream_, |
| 593 | /*chosen_stream=*/0, |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 594 | receiver->transport_.get(), |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 595 | &matcher_) {} |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 596 | |
| 597 | } // namespace test |
| 598 | } // namespace webrtc |