Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | #include "test/scenario/video_stream.h" |
| 11 | |
| 12 | #include <algorithm> |
| 13 | #include <utility> |
| 14 | |
Sebastian Jansson | 9a4f38e | 2018-12-19 12:14:41 | [diff] [blame] | 15 | #include "absl/memory/memory.h" |
Danil Chapovalov | 99b71df | 2018-10-26 13:57:48 | [diff] [blame] | 16 | #include "api/test/video/function_video_encoder_factory.h" |
Jiawei Ou | c2ebe21 | 2018-11-08 18:02:56 | [diff] [blame] | 17 | #include "api/video/builtin_video_bitrate_allocator_factory.h" |
Steve Anton | 10542f2 | 2019-01-11 17:11:00 | [diff] [blame] | 18 | #include "media/base/media_constants.h" |
| 19 | #include "media/engine/internal_decoder_factory.h" |
| 20 | #include "media/engine/internal_encoder_factory.h" |
| 21 | #include "media/engine/webrtc_video_engine.h" |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 22 | #include "test/call_test.h" |
| 23 | #include "test/fake_encoder.h" |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 24 | #include "test/scenario/hardware_codecs.h" |
Steve Anton | 10542f2 | 2019-01-11 17:11:00 | [diff] [blame] | 25 | #include "test/testsupport/file_utils.h" |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 26 | |
| 27 | namespace webrtc { |
| 28 | namespace test { |
| 29 | namespace { |
Elad Alon | d8d3248 | 2019-02-18 22:45:57 | [diff] [blame] | 30 | enum : int { // The first valid value is 1. |
| 31 | kTransportSequenceNumberExtensionId = 1, |
| 32 | kVideoContentTypeExtensionId, |
| 33 | kVideoRotationRtpExtensionId, |
| 34 | }; |
| 35 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 36 | constexpr int kDefaultMaxQp = cricket::WebRtcVideoChannel::kDefaultQpMax; |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 37 | uint8_t CodecTypeToPayloadType(VideoCodecType codec_type) { |
| 38 | switch (codec_type) { |
| 39 | case VideoCodecType::kVideoCodecGeneric: |
| 40 | return CallTest::kFakeVideoSendPayloadType; |
| 41 | case VideoCodecType::kVideoCodecVP8: |
| 42 | return CallTest::kPayloadTypeVP8; |
| 43 | case VideoCodecType::kVideoCodecVP9: |
| 44 | return CallTest::kPayloadTypeVP9; |
| 45 | case VideoCodecType::kVideoCodecH264: |
| 46 | return CallTest::kPayloadTypeH264; |
| 47 | default: |
| 48 | RTC_NOTREACHED(); |
| 49 | } |
| 50 | return {}; |
| 51 | } |
| 52 | std::string CodecTypeToCodecName(VideoCodecType codec_type) { |
| 53 | switch (codec_type) { |
| 54 | case VideoCodecType::kVideoCodecGeneric: |
| 55 | return ""; |
| 56 | case VideoCodecType::kVideoCodecVP8: |
| 57 | return cricket::kVp8CodecName; |
| 58 | case VideoCodecType::kVideoCodecVP9: |
| 59 | return cricket::kVp9CodecName; |
| 60 | case VideoCodecType::kVideoCodecH264: |
| 61 | return cricket::kH264CodecName; |
| 62 | default: |
| 63 | RTC_NOTREACHED(); |
| 64 | } |
| 65 | return {}; |
| 66 | } |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 67 | VideoEncoderConfig::ContentType ConvertContentType( |
| 68 | VideoStreamConfig::Encoder::ContentType content_type) { |
| 69 | switch (content_type) { |
| 70 | case VideoStreamConfig::Encoder::ContentType::kVideo: |
| 71 | return VideoEncoderConfig::ContentType::kRealtimeVideo; |
| 72 | break; |
| 73 | case VideoStreamConfig::Encoder::ContentType::kScreen: |
| 74 | return VideoEncoderConfig::ContentType::kScreen; |
| 75 | } |
| 76 | } |
Sebastian Jansson | 06c5145 | 2019-02-18 15:55:15 | [diff] [blame] | 77 | InterLayerPredMode ToInterLayerPredMode( |
| 78 | VideoStreamConfig::Encoder::Layers::Prediction value) { |
| 79 | using Pred = VideoStreamConfig::Encoder::Layers::Prediction; |
| 80 | switch (value) { |
| 81 | case Pred::kTemporalOnly: |
| 82 | return InterLayerPredMode::kOff; |
| 83 | case Pred::kSpatialOnKey: |
| 84 | return InterLayerPredMode::kOnKeyPic; |
| 85 | case Pred::kFull: |
| 86 | return InterLayerPredMode::kOn; |
| 87 | } |
| 88 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 89 | std::vector<RtpExtension> GetVideoRtpExtensions( |
| 90 | const VideoStreamConfig config) { |
| 91 | return {RtpExtension(RtpExtension::kTransportSequenceNumberUri, |
| 92 | kTransportSequenceNumberExtensionId), |
| 93 | RtpExtension(RtpExtension::kVideoContentTypeUri, |
| 94 | kVideoContentTypeExtensionId), |
| 95 | RtpExtension(RtpExtension::kVideoRotationUri, |
| 96 | kVideoRotationRtpExtensionId)}; |
| 97 | } |
| 98 | |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 99 | std::string TransformFilePath(std::string path) { |
| 100 | static const std::string resource_prefix = "res://"; |
| 101 | int ext_pos = path.rfind("."); |
| 102 | if (ext_pos < 0) { |
| 103 | return test::ResourcePath(path, "yuv"); |
| 104 | } else if (path.find(resource_prefix) == 0) { |
| 105 | std::string name = path.substr(resource_prefix.length(), ext_pos); |
| 106 | std::string ext = path.substr(ext_pos, path.size()); |
| 107 | return test::ResourcePath(name, ext); |
| 108 | } |
| 109 | return path; |
| 110 | } |
| 111 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 112 | VideoSendStream::Config CreateVideoSendStreamConfig(VideoStreamConfig config, |
| 113 | std::vector<uint32_t> ssrcs, |
| 114 | Transport* send_transport) { |
| 115 | VideoSendStream::Config send_config(send_transport); |
| 116 | send_config.rtp.payload_name = CodecTypeToPayloadString(config.encoder.codec); |
| 117 | send_config.rtp.payload_type = CodecTypeToPayloadType(config.encoder.codec); |
| 118 | |
| 119 | send_config.rtp.ssrcs = ssrcs; |
| 120 | send_config.rtp.extensions = GetVideoRtpExtensions(config); |
| 121 | |
| 122 | if (config.stream.use_flexfec) { |
| 123 | send_config.rtp.flexfec.payload_type = CallTest::kFlexfecPayloadType; |
| 124 | send_config.rtp.flexfec.ssrc = CallTest::kFlexfecSendSsrc; |
| 125 | send_config.rtp.flexfec.protected_media_ssrcs = ssrcs; |
| 126 | } |
| 127 | if (config.stream.use_ulpfec) { |
| 128 | send_config.rtp.ulpfec.red_payload_type = CallTest::kRedPayloadType; |
| 129 | send_config.rtp.ulpfec.ulpfec_payload_type = CallTest::kUlpfecPayloadType; |
| 130 | send_config.rtp.ulpfec.red_rtx_payload_type = CallTest::kRtxRedPayloadType; |
| 131 | } |
| 132 | return send_config; |
| 133 | } |
| 134 | rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings> |
Sebastian Jansson | 06c5145 | 2019-02-18 15:55:15 | [diff] [blame] | 135 | CreateVp9SpecificSettings(VideoStreamConfig video_config) { |
| 136 | constexpr auto kScreen = VideoStreamConfig::Encoder::ContentType::kScreen; |
| 137 | VideoStreamConfig::Encoder conf = video_config.encoder; |
| 138 | VideoCodecVP9 vp9 = VideoEncoder::GetDefaultVp9Settings(); |
| 139 | vp9.frameDroppingOn = conf.frame_dropping; |
| 140 | vp9.keyFrameInterval = conf.key_frame_interval.value_or(0); |
| 141 | vp9.numberOfTemporalLayers = static_cast<uint8_t>(conf.layers.temporal); |
| 142 | vp9.numberOfSpatialLayers = static_cast<uint8_t>(conf.layers.spatial); |
| 143 | vp9.interLayerPred = ToInterLayerPredMode(conf.layers.prediction); |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 144 | |
Sebastian Jansson | 06c5145 | 2019-02-18 15:55:15 | [diff] [blame] | 145 | if (conf.content_type == kScreen && |
| 146 | (video_config.source.framerate > 5 || conf.layers.spatial >= 3)) { |
| 147 | vp9.flexibleMode = true; |
| 148 | } |
| 149 | |
| 150 | if (conf.content_type == kScreen || |
| 151 | conf.layers.temporal * conf.layers.spatial) { |
| 152 | vp9.automaticResizeOn = false; |
| 153 | vp9.denoisingOn = false; |
| 154 | } else { |
| 155 | vp9.automaticResizeOn = conf.single.automatic_scaling; |
| 156 | vp9.denoisingOn = conf.single.denoising; |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 157 | } |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 158 | return new rtc::RefCountedObject< |
Sebastian Jansson | 06c5145 | 2019-02-18 15:55:15 | [diff] [blame] | 159 | VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9); |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 160 | } |
| 161 | |
| 162 | rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings> |
| 163 | CreateVp8SpecificSettings(VideoStreamConfig config) { |
| 164 | VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings(); |
| 165 | vp8_settings.frameDroppingOn = config.encoder.frame_dropping; |
| 166 | vp8_settings.keyFrameInterval = config.encoder.key_frame_interval.value_or(0); |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 167 | vp8_settings.numberOfTemporalLayers = config.encoder.layers.temporal; |
| 168 | if (config.encoder.layers.spatial * config.encoder.layers.temporal > 1) { |
| 169 | vp8_settings.automaticResizeOn = false; |
| 170 | vp8_settings.denoisingOn = false; |
| 171 | } else { |
| 172 | vp8_settings.automaticResizeOn = config.encoder.single.automatic_scaling; |
| 173 | vp8_settings.denoisingOn = config.encoder.single.denoising; |
| 174 | } |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 175 | return new rtc::RefCountedObject< |
| 176 | VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings); |
| 177 | } |
| 178 | |
| 179 | rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings> |
| 180 | CreateH264SpecificSettings(VideoStreamConfig config) { |
Sebastian Jansson | 06c5145 | 2019-02-18 15:55:15 | [diff] [blame] | 181 | RTC_DCHECK_EQ(config.encoder.layers.temporal, 1); |
| 182 | RTC_DCHECK_EQ(config.encoder.layers.spatial, 1); |
| 183 | |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 184 | VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings(); |
| 185 | h264_settings.frameDroppingOn = config.encoder.frame_dropping; |
| 186 | h264_settings.keyFrameInterval = |
| 187 | config.encoder.key_frame_interval.value_or(0); |
| 188 | return new rtc::RefCountedObject< |
| 189 | VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings); |
| 190 | } |
| 191 | |
| 192 | rtc::scoped_refptr<VideoEncoderConfig::EncoderSpecificSettings> |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 193 | CreateEncoderSpecificSettings(VideoStreamConfig config) { |
| 194 | using Codec = VideoStreamConfig::Encoder::Codec; |
| 195 | switch (config.encoder.codec) { |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 196 | case Codec::kVideoCodecH264: |
| 197 | return CreateH264SpecificSettings(config); |
| 198 | case Codec::kVideoCodecVP8: |
| 199 | return CreateVp8SpecificSettings(config); |
| 200 | case Codec::kVideoCodecVP9: |
| 201 | return CreateVp9SpecificSettings(config); |
| 202 | case Codec::kVideoCodecGeneric: |
| 203 | return nullptr; |
| 204 | case Codec::kVideoCodecMultiplex: |
| 205 | RTC_NOTREACHED(); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 206 | return nullptr; |
| 207 | } |
| 208 | } |
| 209 | |
| 210 | VideoEncoderConfig CreateVideoEncoderConfig(VideoStreamConfig config) { |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 211 | VideoEncoderConfig encoder_config; |
| 212 | encoder_config.codec_type = config.encoder.codec; |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 213 | encoder_config.content_type = ConvertContentType(config.encoder.content_type); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 214 | encoder_config.video_format = |
| 215 | SdpVideoFormat(CodecTypeToPayloadString(config.encoder.codec), {}); |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 216 | |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 217 | size_t num_streams = 1; |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 218 | if (config.encoder.codec == VideoStreamConfig::Encoder::Codec::kVideoCodecVP8) |
| 219 | num_streams = static_cast<size_t>(config.encoder.layers.spatial); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 220 | encoder_config.number_of_streams = num_streams; |
| 221 | encoder_config.simulcast_layers = std::vector<VideoStream>(num_streams); |
Sebastian Jansson | 985ee68 | 2018-11-12 15:33:29 | [diff] [blame] | 222 | encoder_config.min_transmit_bitrate_bps = config.stream.pad_to_rate.bps(); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 223 | |
| 224 | std::string cricket_codec = CodecTypeToCodecName(config.encoder.codec); |
| 225 | if (!cricket_codec.empty()) { |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 226 | bool screenshare = config.encoder.content_type == |
| 227 | VideoStreamConfig::Encoder::ContentType::kScreen; |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 228 | encoder_config.video_stream_factory = |
| 229 | new rtc::RefCountedObject<cricket::EncoderStreamFactory>( |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 230 | cricket_codec, kDefaultMaxQp, screenshare, screenshare); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 231 | } else { |
| 232 | encoder_config.video_stream_factory = |
| 233 | new rtc::RefCountedObject<DefaultVideoStreamFactory>(); |
| 234 | } |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 235 | |
| 236 | // TODO(srte): Base this on encoder capabilities. |
| 237 | encoder_config.max_bitrate_bps = |
| 238 | config.encoder.max_data_rate.value_or(DataRate::kbps(10000)).bps(); |
| 239 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 240 | encoder_config.encoder_specific_settings = |
| 241 | CreateEncoderSpecificSettings(config); |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 242 | if (config.encoder.max_framerate) { |
| 243 | for (auto& layer : encoder_config.simulcast_layers) { |
| 244 | layer.max_framerate = *config.encoder.max_framerate; |
| 245 | } |
| 246 | } |
| 247 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 248 | return encoder_config; |
| 249 | } |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 250 | |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 251 | std::unique_ptr<FrameGenerator> CreateImageSlideGenerator( |
| 252 | Clock* clock, |
| 253 | VideoStreamConfig::Source::Slides slides, |
| 254 | int framerate) { |
| 255 | std::vector<std::string> paths = slides.images.paths; |
| 256 | for (std::string& path : paths) |
| 257 | path = TransformFilePath(path); |
| 258 | if (slides.images.crop.width || slides.images.crop.height) { |
| 259 | TimeDelta pause_duration = |
| 260 | slides.change_interval - slides.images.crop.scroll_duration; |
Sebastian Jansson | 5a00016 | 2019-04-12 09:21:32 | [diff] [blame^] | 261 | RTC_CHECK_GE(pause_duration, TimeDelta::Zero()); |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 262 | int crop_width = slides.images.crop.width.value_or(slides.images.width); |
| 263 | int crop_height = slides.images.crop.height.value_or(slides.images.height); |
| 264 | RTC_CHECK_LE(crop_width, slides.images.width); |
| 265 | RTC_CHECK_LE(crop_height, slides.images.height); |
| 266 | return FrameGenerator::CreateScrollingInputFromYuvFiles( |
| 267 | clock, paths, slides.images.width, slides.images.height, crop_width, |
| 268 | crop_height, slides.images.crop.scroll_duration.ms(), |
| 269 | pause_duration.ms()); |
| 270 | } else { |
| 271 | return FrameGenerator::CreateFromYuvFile( |
| 272 | paths, slides.images.width, slides.images.height, |
| 273 | slides.change_interval.seconds<double>() * framerate); |
| 274 | } |
| 275 | } |
| 276 | |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 277 | std::unique_ptr<FrameGenerator> CreateFrameGenerator( |
| 278 | Clock* clock, |
| 279 | VideoStreamConfig::Source source) { |
| 280 | using Capture = VideoStreamConfig::Source::Capture; |
| 281 | switch (source.capture) { |
| 282 | case Capture::kGenerator: |
| 283 | return FrameGenerator::CreateSquareGenerator( |
| 284 | source.generator.width, source.generator.height, |
| 285 | source.generator.pixel_format, /*num_squares*/ absl::nullopt); |
| 286 | case Capture::kVideoFile: |
| 287 | RTC_CHECK(source.video_file.width && source.video_file.height); |
| 288 | return FrameGenerator::CreateFromYuvFile( |
Sebastian Jansson | d37307c | 2019-02-22 16:07:49 | [diff] [blame] | 289 | {TransformFilePath(source.video_file.name)}, source.video_file.width, |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 290 | source.video_file.height, /*frame_repeat_count*/ 1); |
Sebastian Jansson | f2727fb | 2019-02-18 15:54:55 | [diff] [blame] | 291 | case Capture::kGenerateSlides: |
| 292 | return FrameGenerator::CreateSlideGenerator( |
| 293 | source.slides.generator.width, source.slides.generator.height, |
| 294 | source.slides.change_interval.seconds<double>() * source.framerate); |
| 295 | case Capture::kImageSlides: |
| 296 | return CreateImageSlideGenerator(clock, source.slides, source.framerate); |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 297 | } |
| 298 | } |
| 299 | |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 300 | VideoReceiveStream::Config CreateVideoReceiveStreamConfig( |
| 301 | VideoStreamConfig config, |
| 302 | Transport* feedback_transport, |
| 303 | VideoReceiveStream::Decoder decoder, |
| 304 | rtc::VideoSinkInterface<VideoFrame>* renderer, |
| 305 | uint32_t local_ssrc, |
| 306 | uint32_t ssrc, |
| 307 | uint32_t rtx_ssrc) { |
| 308 | VideoReceiveStream::Config recv(feedback_transport); |
| 309 | recv.rtp.remb = !config.stream.packet_feedback; |
| 310 | recv.rtp.transport_cc = config.stream.packet_feedback; |
| 311 | recv.rtp.local_ssrc = local_ssrc; |
| 312 | recv.rtp.extensions = GetVideoRtpExtensions(config); |
| 313 | RTC_DCHECK(!config.stream.use_rtx || |
| 314 | config.stream.nack_history_time > TimeDelta::Zero()); |
| 315 | recv.rtp.nack.rtp_history_ms = config.stream.nack_history_time.ms(); |
| 316 | recv.rtp.protected_by_flexfec = config.stream.use_flexfec; |
| 317 | recv.rtp.remote_ssrc = ssrc; |
| 318 | recv.decoders.push_back(decoder); |
| 319 | recv.renderer = renderer; |
| 320 | if (config.stream.use_rtx) { |
| 321 | recv.rtp.rtx_ssrc = rtx_ssrc; |
| 322 | recv.rtp.rtx_associated_payload_types[CallTest::kSendRtxPayloadType] = |
| 323 | CodecTypeToPayloadType(config.encoder.codec); |
| 324 | } |
| 325 | if (config.stream.use_ulpfec) { |
| 326 | recv.rtp.red_payload_type = CallTest::kRedPayloadType; |
| 327 | recv.rtp.ulpfec_payload_type = CallTest::kUlpfecPayloadType; |
| 328 | recv.rtp.rtx_associated_payload_types[CallTest::kRtxRedPayloadType] = |
| 329 | CallTest::kRedPayloadType; |
| 330 | } |
Sebastian Jansson | cc5be54 | 2019-04-01 11:34:55 | [diff] [blame] | 331 | recv.sync_group = config.render.sync_group; |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 332 | return recv; |
| 333 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 334 | } // namespace |
| 335 | |
| 336 | SendVideoStream::SendVideoStream(CallClient* sender, |
| 337 | VideoStreamConfig config, |
Sebastian Jansson | 9a4f38e | 2018-12-19 12:14:41 | [diff] [blame] | 338 | Transport* send_transport, |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 339 | VideoFrameMatcher* matcher) |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 340 | : sender_(sender), config_(config) { |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 341 | video_capturer_ = absl::make_unique<FrameGeneratorCapturer>( |
| 342 | sender_->clock_, CreateFrameGenerator(sender_->clock_, config.source), |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 343 | config.source.framerate, |
| 344 | *sender->time_controller_->GetTaskQueueFactory()); |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 345 | video_capturer_->Init(); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 346 | |
| 347 | using Encoder = VideoStreamConfig::Encoder; |
| 348 | using Codec = VideoStreamConfig::Encoder::Codec; |
| 349 | switch (config.encoder.implementation) { |
| 350 | case Encoder::Implementation::kFake: |
| 351 | if (config.encoder.codec == Codec::kVideoCodecGeneric) { |
| 352 | encoder_factory_ = |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 353 | absl::make_unique<FunctionVideoEncoderFactory>([this]() { |
| 354 | rtc::CritScope cs(&crit_); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 355 | auto encoder = |
| 356 | absl::make_unique<test::FakeEncoder>(sender_->clock_); |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 357 | fake_encoders_.push_back(encoder.get()); |
| 358 | if (config_.encoder.fake.max_rate.IsFinite()) |
| 359 | encoder->SetMaxBitrate(config_.encoder.fake.max_rate.kbps()); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 360 | return encoder; |
| 361 | }); |
| 362 | } else { |
| 363 | RTC_NOTREACHED(); |
| 364 | } |
| 365 | break; |
| 366 | case VideoStreamConfig::Encoder::Implementation::kSoftware: |
| 367 | encoder_factory_.reset(new InternalEncoderFactory()); |
| 368 | break; |
| 369 | case VideoStreamConfig::Encoder::Implementation::kHardware: |
| 370 | encoder_factory_ = CreateHardwareEncoderFactory(); |
| 371 | break; |
| 372 | } |
| 373 | RTC_CHECK(encoder_factory_); |
| 374 | |
Jiawei Ou | c2ebe21 | 2018-11-08 18:02:56 | [diff] [blame] | 375 | bitrate_allocator_factory_ = CreateBuiltinVideoBitrateAllocatorFactory(); |
| 376 | RTC_CHECK(bitrate_allocator_factory_); |
| 377 | |
Sebastian Jansson | f0c366b | 2019-02-14 12:18:42 | [diff] [blame] | 378 | VideoEncoderConfig encoder_config = CreateVideoEncoderConfig(config); |
| 379 | for (size_t i = 0; i < encoder_config.number_of_streams; ++i) { |
| 380 | ssrcs_.push_back(sender->GetNextVideoSsrc()); |
| 381 | rtx_ssrcs_.push_back(sender->GetNextRtxSsrc()); |
| 382 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 383 | VideoSendStream::Config send_config = |
| 384 | CreateVideoSendStreamConfig(config, ssrcs_, send_transport); |
| 385 | send_config.encoder_settings.encoder_factory = encoder_factory_.get(); |
Jiawei Ou | c2ebe21 | 2018-11-08 18:02:56 | [diff] [blame] | 386 | send_config.encoder_settings.bitrate_allocator_factory = |
| 387 | bitrate_allocator_factory_.get(); |
| 388 | |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 389 | sender_->SendTask([&] { |
Sebastian Jansson | cc5be54 | 2019-04-01 11:34:55 | [diff] [blame] | 390 | if (config.stream.fec_controller_factory) { |
| 391 | send_stream_ = sender_->call_->CreateVideoSendStream( |
| 392 | std::move(send_config), std::move(encoder_config), |
| 393 | config.stream.fec_controller_factory->CreateFecController()); |
| 394 | } else { |
| 395 | send_stream_ = sender_->call_->CreateVideoSendStream( |
| 396 | std::move(send_config), std::move(encoder_config)); |
| 397 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 398 | |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 399 | if (matcher->Active()) { |
| 400 | frame_tap_ = absl::make_unique<ForwardingCapturedFrameTap>( |
| 401 | sender_->clock_, matcher, video_capturer_.get()); |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 402 | send_stream_->SetSource(frame_tap_.get(), |
| 403 | config.encoder.degradation_preference); |
| 404 | } else { |
| 405 | send_stream_->SetSource(video_capturer_.get(), |
| 406 | config.encoder.degradation_preference); |
| 407 | } |
| 408 | }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 409 | } |
| 410 | |
| 411 | SendVideoStream::~SendVideoStream() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 412 | sender_->SendTask( |
| 413 | [this] { sender_->call_->DestroyVideoSendStream(send_stream_); }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 414 | } |
| 415 | |
| 416 | void SendVideoStream::Start() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 417 | sender_->SendTask([this] { |
| 418 | send_stream_->Start(); |
| 419 | sender_->call_->SignalChannelNetworkState(MediaType::VIDEO, kNetworkUp); |
| 420 | }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 421 | } |
| 422 | |
Christoffer Rodbro | 5f6abcf | 2019-02-08 10:00:10 | [diff] [blame] | 423 | void SendVideoStream::Stop() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 424 | sender_->SendTask([this] { send_stream_->Stop(); }); |
Christoffer Rodbro | 5f6abcf | 2019-02-08 10:00:10 | [diff] [blame] | 425 | } |
| 426 | |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 427 | void SendVideoStream::UpdateConfig( |
| 428 | std::function<void(VideoStreamConfig*)> modifier) { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 429 | sender_->SendTask([&] { |
| 430 | rtc::CritScope cs(&crit_); |
| 431 | VideoStreamConfig prior_config = config_; |
| 432 | modifier(&config_); |
| 433 | if (prior_config.encoder.fake.max_rate != config_.encoder.fake.max_rate) { |
| 434 | for (auto* encoder : fake_encoders_) { |
| 435 | encoder->SetMaxBitrate(config_.encoder.fake.max_rate.kbps()); |
| 436 | } |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 437 | } |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 438 | // TODO(srte): Add more conditions that should cause reconfiguration. |
| 439 | if (prior_config.encoder.max_framerate != config_.encoder.max_framerate) { |
| 440 | VideoEncoderConfig encoder_config = CreateVideoEncoderConfig(config_); |
| 441 | send_stream_->ReconfigureVideoEncoder(std::move(encoder_config)); |
| 442 | } |
| 443 | if (prior_config.source.framerate != config_.source.framerate) { |
| 444 | SetCaptureFramerate(config_.source.framerate); |
| 445 | } |
| 446 | }); |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 447 | } |
| 448 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 449 | void SendVideoStream::SetCaptureFramerate(int framerate) { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 450 | sender_->SendTask([&] { video_capturer_->ChangeFramerate(framerate); }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 451 | } |
| 452 | |
| 453 | VideoSendStream::Stats SendVideoStream::GetStats() const { |
| 454 | return send_stream_->GetStats(); |
| 455 | } |
| 456 | |
| 457 | ColumnPrinter SendVideoStream::StatsPrinter() { |
| 458 | return ColumnPrinter::Lambda( |
| 459 | "video_target_rate video_sent_rate width height", |
| 460 | [this](rtc::SimpleStringBuilder& sb) { |
| 461 | VideoSendStream::Stats video_stats = send_stream_->GetStats(); |
| 462 | int width = 0; |
| 463 | int height = 0; |
Mirko Bonadei | 739baf0 | 2019-01-27 16:29:42 | [diff] [blame] | 464 | for (const auto& stream_stat : video_stats.substreams) { |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 465 | width = std::max(width, stream_stat.second.width); |
| 466 | height = std::max(height, stream_stat.second.height); |
| 467 | } |
| 468 | sb.AppendFormat("%.0lf %.0lf %i %i", |
| 469 | video_stats.target_media_bitrate_bps / 8.0, |
| 470 | video_stats.media_bitrate_bps / 8.0, width, height); |
| 471 | }, |
| 472 | 64); |
| 473 | } |
| 474 | |
| 475 | ReceiveVideoStream::ReceiveVideoStream(CallClient* receiver, |
| 476 | VideoStreamConfig config, |
| 477 | SendVideoStream* send_stream, |
| 478 | size_t chosen_stream, |
Sebastian Jansson | 9a4f38e | 2018-12-19 12:14:41 | [diff] [blame] | 479 | Transport* feedback_transport, |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 480 | VideoFrameMatcher* matcher) |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 481 | : receiver_(receiver), config_(config) { |
Sebastian Jansson | 0c32e33 | 2018-11-12 15:47:43 | [diff] [blame] | 482 | if (config.encoder.codec == |
| 483 | VideoStreamConfig::Encoder::Codec::kVideoCodecGeneric) { |
| 484 | decoder_factory_ = absl::make_unique<FunctionVideoDecoderFactory>( |
| 485 | []() { return absl::make_unique<FakeDecoder>(); }); |
| 486 | } else { |
| 487 | decoder_factory_ = absl::make_unique<InternalDecoderFactory>(); |
| 488 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 489 | |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 490 | VideoReceiveStream::Decoder decoder = |
| 491 | CreateMatchingDecoder(CodecTypeToPayloadType(config.encoder.codec), |
| 492 | CodecTypeToPayloadString(config.encoder.codec)); |
| 493 | decoder.decoder_factory = decoder_factory_.get(); |
| 494 | size_t num_streams = 1; |
| 495 | if (config.encoder.codec == VideoStreamConfig::Encoder::Codec::kVideoCodecVP8) |
| 496 | num_streams = config.encoder.layers.spatial; |
| 497 | for (size_t i = 0; i < num_streams; ++i) { |
| 498 | rtc::VideoSinkInterface<VideoFrame>* renderer = &fake_renderer_; |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 499 | if (matcher->Active()) { |
| 500 | render_taps_.emplace_back(absl::make_unique<DecodedFrameTap>(matcher, i)); |
| 501 | renderer = render_taps_.back().get(); |
Sebastian Jansson | 5fbebd5 | 2019-02-20 10:16:19 | [diff] [blame] | 502 | } |
| 503 | auto recv_config = CreateVideoReceiveStreamConfig( |
| 504 | config, feedback_transport, decoder, renderer, |
| 505 | receiver_->GetNextVideoLocalSsrc(), send_stream->ssrcs_[i], |
| 506 | send_stream->rtx_ssrcs_[i]); |
| 507 | if (config.stream.use_flexfec) { |
| 508 | RTC_DCHECK(num_streams == 1); |
| 509 | FlexfecReceiveStream::Config flexfec(feedback_transport); |
| 510 | flexfec.payload_type = CallTest::kFlexfecPayloadType; |
| 511 | flexfec.remote_ssrc = CallTest::kFlexfecSendSsrc; |
| 512 | flexfec.protected_media_ssrcs = send_stream->rtx_ssrcs_; |
| 513 | flexfec.local_ssrc = recv_config.rtp.local_ssrc; |
| 514 | receiver_->ssrc_media_types_[flexfec.remote_ssrc] = MediaType::VIDEO; |
| 515 | flecfec_stream_ = receiver_->call_->CreateFlexfecReceiveStream(flexfec); |
| 516 | } |
| 517 | receiver_->ssrc_media_types_[recv_config.rtp.remote_ssrc] = |
| 518 | MediaType::VIDEO; |
| 519 | if (config.stream.use_rtx) |
| 520 | receiver_->ssrc_media_types_[recv_config.rtp.rtx_ssrc] = MediaType::VIDEO; |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 521 | receiver_->SendTask([this, &recv_config] { |
| 522 | receive_streams_.push_back( |
| 523 | receiver_->call_->CreateVideoReceiveStream(std::move(recv_config))); |
| 524 | }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 525 | } |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 526 | } |
| 527 | |
| 528 | ReceiveVideoStream::~ReceiveVideoStream() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 529 | receiver_->SendTask([this] { |
| 530 | for (auto* recv_stream : receive_streams_) |
| 531 | receiver_->call_->DestroyVideoReceiveStream(recv_stream); |
| 532 | if (flecfec_stream_) |
| 533 | receiver_->call_->DestroyFlexfecReceiveStream(flecfec_stream_); |
| 534 | }); |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 535 | } |
| 536 | |
Sebastian Jansson | 49a7843 | 2018-11-20 15:15:29 | [diff] [blame] | 537 | void ReceiveVideoStream::Start() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 538 | receiver_->SendTask([this] { |
| 539 | for (auto* recv_stream : receive_streams_) |
| 540 | recv_stream->Start(); |
| 541 | receiver_->call_->SignalChannelNetworkState(MediaType::VIDEO, kNetworkUp); |
| 542 | }); |
Sebastian Jansson | 49a7843 | 2018-11-20 15:15:29 | [diff] [blame] | 543 | } |
| 544 | |
Sebastian Jansson | bdfadd6 | 2019-02-08 12:34:57 | [diff] [blame] | 545 | void ReceiveVideoStream::Stop() { |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 546 | receiver_->SendTask([this] { |
| 547 | for (auto* recv_stream : receive_streams_) |
| 548 | recv_stream->Stop(); |
| 549 | }); |
Sebastian Jansson | bdfadd6 | 2019-02-08 12:34:57 | [diff] [blame] | 550 | } |
| 551 | |
Sebastian Jansson | f4481c8 | 2019-04-09 10:48:34 | [diff] [blame] | 552 | VideoReceiveStream::Stats ReceiveVideoStream::GetStats() const { |
| 553 | if (receive_streams_.empty()) |
| 554 | return VideoReceiveStream::Stats(); |
| 555 | // TODO(srte): Handle multiple receive streams. |
| 556 | return receive_streams_.front()->GetStats(); |
| 557 | } |
| 558 | |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 559 | VideoStreamPair::~VideoStreamPair() = default; |
| 560 | |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 561 | VideoStreamPair::VideoStreamPair(CallClient* sender, |
| 562 | CallClient* receiver, |
| 563 | VideoStreamConfig config) |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 564 | : config_(config), |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 565 | matcher_(config.hooks.frame_pair_handlers), |
| 566 | send_stream_(sender, config, sender->transport_.get(), &matcher_), |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 567 | receive_stream_(receiver, |
| 568 | config, |
| 569 | &send_stream_, |
| 570 | /*chosen_stream=*/0, |
Sebastian Jansson | 105a10a | 2019-04-01 07:18:14 | [diff] [blame] | 571 | receiver->transport_.get(), |
Sebastian Jansson | cf2df2f | 2019-04-02 09:51:28 | [diff] [blame] | 572 | &matcher_) {} |
Sebastian Jansson | 98b07e91 | 2018-09-27 11:47:01 | [diff] [blame] | 573 | |
| 574 | } // namespace test |
| 575 | } // namespace webrtc |