blob: 799e366ba31125dd6916e268c9501b77f0280bb5 [file] [log] [blame]
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/video_send_stream.h"
#include <utility>
#include "api/array_view.h"
#include "api/task_queue/task_queue_base.h"
#include "api/video/video_stream_encoder_settings.h"
#include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
#include "modules/rtp_rtcp/source/rtp_header_extension_size.h"
#include "modules/rtp_rtcp/source/rtp_sender.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/strings/string_builder.h"
#include "system_wrappers/include/clock.h"
#include "video/adaptation/overuse_frame_detector.h"
#include "video/frame_cadence_adapter.h"
#include "video/video_stream_encoder.h"
namespace webrtc {
namespace {
size_t CalculateMaxHeaderSize(const RtpConfig& config) {
size_t header_size = kRtpHeaderSize;
size_t extensions_size = 0;
size_t fec_extensions_size = 0;
if (!config.extensions.empty()) {
RtpHeaderExtensionMap extensions_map(config.extensions);
extensions_size = RtpHeaderExtensionSize(RTPSender::VideoExtensionSizes(),
extensions_map);
fec_extensions_size =
RtpHeaderExtensionSize(RTPSender::FecExtensionSizes(), extensions_map);
}
header_size += extensions_size;
if (config.flexfec.payload_type >= 0) {
// All FEC extensions again plus maximum FlexFec overhead.
header_size += fec_extensions_size + 32;
} else {
if (config.ulpfec.ulpfec_payload_type >= 0) {
// Header with all the FEC extensions will be repeated plus maximum
// UlpFec overhead.
header_size += fec_extensions_size + 18;
}
if (config.ulpfec.red_payload_type >= 0) {
header_size += 1; // RED header.
}
}
// Additional room for Rtx.
if (config.rtx.payload_type >= 0)
header_size += kRtxHeaderSize;
return header_size;
}
VideoStreamEncoder::BitrateAllocationCallbackType
GetBitrateAllocationCallbackType(const VideoSendStream::Config& config,
const FieldTrialsView& field_trials) {
if (webrtc::RtpExtension::FindHeaderExtensionByUri(
config.rtp.extensions,
webrtc::RtpExtension::kVideoLayersAllocationUri,
config.crypto_options.srtp.enable_encrypted_rtp_header_extensions
? RtpExtension::Filter::kPreferEncryptedExtension
: RtpExtension::Filter::kDiscardEncryptedExtension)) {
return VideoStreamEncoder::BitrateAllocationCallbackType::
kVideoLayersAllocation;
}
if (field_trials.IsEnabled("WebRTC-Target-Bitrate-Rtcp")) {
return VideoStreamEncoder::BitrateAllocationCallbackType::
kVideoBitrateAllocation;
}
return VideoStreamEncoder::BitrateAllocationCallbackType::
kVideoBitrateAllocationWhenScreenSharing;
}
RtpSenderFrameEncryptionConfig CreateFrameEncryptionConfig(
const VideoSendStream::Config* config) {
RtpSenderFrameEncryptionConfig frame_encryption_config;
frame_encryption_config.frame_encryptor = config->frame_encryptor.get();
frame_encryption_config.crypto_options = config->crypto_options;
return frame_encryption_config;
}
RtpSenderObservers CreateObservers(RtcpRttStats* call_stats,
EncoderRtcpFeedback* encoder_feedback,
SendStatisticsProxy* stats_proxy,
SendDelayStats* send_delay_stats) {
RtpSenderObservers observers;
observers.rtcp_rtt_stats = call_stats;
observers.intra_frame_callback = encoder_feedback;
observers.rtcp_loss_notification_observer = encoder_feedback;
observers.report_block_data_observer = stats_proxy;
observers.rtp_stats = stats_proxy;
observers.bitrate_observer = stats_proxy;
observers.frame_count_observer = stats_proxy;
observers.rtcp_type_observer = stats_proxy;
observers.send_delay_observer = stats_proxy;
observers.send_packet_observer = send_delay_stats;
return observers;
}
std::unique_ptr<VideoStreamEncoder> CreateVideoStreamEncoder(
Clock* clock,
int num_cpu_cores,
TaskQueueFactory* task_queue_factory,
SendStatisticsProxy* stats_proxy,
const VideoStreamEncoderSettings& encoder_settings,
VideoStreamEncoder::BitrateAllocationCallbackType
bitrate_allocation_callback_type,
const FieldTrialsView& field_trials,
webrtc::VideoEncoderFactory::EncoderSelectorInterface* encoder_selector) {
std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue =
task_queue_factory->CreateTaskQueue("EncoderQueue",
TaskQueueFactory::Priority::NORMAL);
TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
return std::make_unique<VideoStreamEncoder>(
clock, num_cpu_cores, stats_proxy, encoder_settings,
std::make_unique<OveruseFrameDetector>(stats_proxy, field_trials),
FrameCadenceAdapterInterface::Create(clock, encoder_queue_ptr,
field_trials),
std::move(encoder_queue), bitrate_allocation_callback_type, field_trials,
encoder_selector);
}
} // namespace
namespace internal {
VideoSendStream::VideoSendStream(
Clock* clock,
int num_cpu_cores,
TaskQueueFactory* task_queue_factory,
TaskQueueBase* network_queue,
RtcpRttStats* call_stats,
RtpTransportControllerSendInterface* transport,
BitrateAllocatorInterface* bitrate_allocator,
SendDelayStats* send_delay_stats,
RtcEventLog* event_log,
VideoSendStream::Config config,
VideoEncoderConfig encoder_config,
const std::map<uint32_t, RtpState>& suspended_ssrcs,
const std::map<uint32_t, RtpPayloadState>& suspended_payload_states,
std::unique_ptr<FecController> fec_controller,
const FieldTrialsView& field_trials)
: rtp_transport_queue_(transport->GetWorkerQueue()->Get()),
transport_(transport),
stats_proxy_(clock, config, encoder_config.content_type, field_trials),
config_(std::move(config)),
content_type_(encoder_config.content_type),
video_stream_encoder_(CreateVideoStreamEncoder(
clock,
num_cpu_cores,
task_queue_factory,
&stats_proxy_,
config_.encoder_settings,
GetBitrateAllocationCallbackType(config_, field_trials),
field_trials,
config_.encoder_selector)),
encoder_feedback_(
clock,
config_.rtp.ssrcs,
video_stream_encoder_.get(),
[this](uint32_t ssrc, const std::vector<uint16_t>& seq_nums) {
return rtp_video_sender_->GetSentRtpPacketInfos(ssrc, seq_nums);
}),
rtp_video_sender_(
transport->CreateRtpVideoSender(suspended_ssrcs,
suspended_payload_states,
config_.rtp,
config_.rtcp_report_interval_ms,
config_.send_transport,
CreateObservers(call_stats,
&encoder_feedback_,
&stats_proxy_,
send_delay_stats),
event_log,
std::move(fec_controller),
CreateFrameEncryptionConfig(&config_),
config_.frame_transformer)),
send_stream_(clock,
&stats_proxy_,
rtp_transport_queue_,
transport,
bitrate_allocator,
video_stream_encoder_.get(),
&config_,
encoder_config.max_bitrate_bps,
encoder_config.bitrate_priority,
encoder_config.content_type,
rtp_video_sender_,
field_trials) {
RTC_DCHECK(config_.encoder_settings.encoder_factory);
RTC_DCHECK(config_.encoder_settings.bitrate_allocator_factory);
video_stream_encoder_->SetFecControllerOverride(rtp_video_sender_);
ReconfigureVideoEncoder(std::move(encoder_config));
}
VideoSendStream::~VideoSendStream() {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_DCHECK(!running_);
transport_->DestroyRtpVideoSender(rtp_video_sender_);
}
void VideoSendStream::UpdateActiveSimulcastLayers(
const std::vector<bool> active_layers) {
RTC_DCHECK_RUN_ON(&thread_checker_);
// Keep our `running_` flag expected state in sync with active layers since
// the `send_stream_` will be implicitly stopped/started depending on the
// state of the layers.
bool running = false;
rtc::StringBuilder active_layers_string;
active_layers_string << "{";
for (size_t i = 0; i < active_layers.size(); ++i) {
if (active_layers[i]) {
running = true;
active_layers_string << "1";
} else {
active_layers_string << "0";
}
if (i < active_layers.size() - 1) {
active_layers_string << ", ";
}
}
active_layers_string << "}";
RTC_LOG(LS_INFO) << "UpdateActiveSimulcastLayers: "
<< active_layers_string.str();
rtp_transport_queue_->PostTask(
SafeTask(transport_queue_safety_, [this, active_layers] {
send_stream_.UpdateActiveSimulcastLayers(active_layers);
}));
running_ = running;
}
void VideoSendStream::Start() {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_DLOG(LS_INFO) << "VideoSendStream::Start";
if (running_)
return;
running_ = true;
rtp_transport_queue_->PostTask([this] {
transport_queue_safety_->SetAlive();
send_stream_.Start();
thread_sync_event_.Set();
});
// It is expected that after VideoSendStream::Start has been called, incoming
// frames are not dropped in VideoStreamEncoder. To ensure this, Start has to
// be synchronized.
// TODO(tommi): ^^^ Validate if this still holds.
thread_sync_event_.Wait(rtc::Event::kForever);
}
void VideoSendStream::Stop() {
RTC_DCHECK_RUN_ON(&thread_checker_);
if (!running_)
return;
RTC_DLOG(LS_INFO) << "VideoSendStream::Stop";
running_ = false;
rtp_transport_queue_->PostTask(SafeTask(transport_queue_safety_, [this] {
// As the stream can get re-used and implicitly restarted via changing
// the state of the active layers, we do not mark the
// `transport_queue_safety_` flag with `SetNotAlive()` here. That's only
// done when we stop permanently via `StopPermanentlyAndGetRtpStates()`.
send_stream_.Stop();
}));
}
bool VideoSendStream::started() {
RTC_DCHECK_RUN_ON(&thread_checker_);
return running_;
}
void VideoSendStream::AddAdaptationResource(
rtc::scoped_refptr<Resource> resource) {
RTC_DCHECK_RUN_ON(&thread_checker_);
video_stream_encoder_->AddAdaptationResource(resource);
}
std::vector<rtc::scoped_refptr<Resource>>
VideoSendStream::GetAdaptationResources() {
RTC_DCHECK_RUN_ON(&thread_checker_);
return video_stream_encoder_->GetAdaptationResources();
}
void VideoSendStream::SetSource(
rtc::VideoSourceInterface<webrtc::VideoFrame>* source,
const DegradationPreference& degradation_preference) {
RTC_DCHECK_RUN_ON(&thread_checker_);
video_stream_encoder_->SetSource(source, degradation_preference);
}
void VideoSendStream::ReconfigureVideoEncoder(VideoEncoderConfig config) {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_DCHECK_EQ(content_type_, config.content_type);
video_stream_encoder_->ConfigureEncoder(
std::move(config),
config_.rtp.max_packet_size - CalculateMaxHeaderSize(config_.rtp));
}
VideoSendStream::Stats VideoSendStream::GetStats() {
// TODO(perkj, solenberg): Some test cases in EndToEndTest call GetStats from
// a network thread. See comment in Call::GetStats().
// RTC_DCHECK_RUN_ON(&thread_checker_);
return stats_proxy_.GetStats();
}
absl::optional<float> VideoSendStream::GetPacingFactorOverride() const {
return send_stream_.configured_pacing_factor();
}
void VideoSendStream::StopPermanentlyAndGetRtpStates(
VideoSendStream::RtpStateMap* rtp_state_map,
VideoSendStream::RtpPayloadStateMap* payload_state_map) {
RTC_DCHECK_RUN_ON(&thread_checker_);
video_stream_encoder_->Stop();
running_ = false;
// Always run these cleanup steps regardless of whether running_ was set
// or not. This will unregister callbacks before destruction.
// See `VideoSendStreamImpl::StopVideoSendStream` for more.
rtp_transport_queue_->PostTask([this, rtp_state_map, payload_state_map]() {
transport_queue_safety_->SetNotAlive();
send_stream_.Stop();
*rtp_state_map = send_stream_.GetRtpStates();
*payload_state_map = send_stream_.GetRtpPayloadStates();
thread_sync_event_.Set();
});
thread_sync_event_.Wait(rtc::Event::kForever);
}
void VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
// Called on a network thread.
send_stream_.DeliverRtcp(packet, length);
}
} // namespace internal
} // namespace webrtc