Remove legacy RtpVideoStreamReceiver.
Bug: none
Change-Id: I434a56980f4d6c68381abae973cd846c71441b08
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/236846
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#37177}
diff --git a/video/BUILD.gn b/video/BUILD.gn
index 74eb3bb..737e028 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -172,8 +172,6 @@
"call_stats.h",
"receive_statistics_proxy.cc",
"receive_statistics_proxy.h",
- "rtp_video_stream_receiver.cc",
- "rtp_video_stream_receiver.h",
"video_quality_observer.cc",
"video_quality_observer.h",
]
@@ -1007,7 +1005,6 @@
sources = [
"call_stats_unittest.cc",
"receive_statistics_proxy_unittest.cc",
- "rtp_video_stream_receiver_unittest.cc",
]
deps = [
":video_legacy",
diff --git a/video/rtp_video_stream_receiver.cc b/video/rtp_video_stream_receiver.cc
deleted file mode 100644
index b09e5fc..0000000
--- a/video/rtp_video_stream_receiver.cc
+++ /dev/null
@@ -1,1278 +0,0 @@
-/*
- * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "video/rtp_video_stream_receiver.h"
-
-#include <algorithm>
-#include <limits>
-#include <memory>
-#include <utility>
-#include <vector>
-
-#include "absl/algorithm/container.h"
-#include "absl/memory/memory.h"
-#include "absl/types/optional.h"
-#include "api/field_trials_view.h"
-#include "media/base/media_constants.h"
-#include "modules/pacing/packet_router.h"
-#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
-#include "modules/rtp_rtcp/include/receive_statistics.h"
-#include "modules/rtp_rtcp/include/rtp_cvo.h"
-#include "modules/rtp_rtcp/include/ulpfec_receiver.h"
-#include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h"
-#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
-#include "modules/rtp_rtcp/source/rtp_format.h"
-#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h"
-#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
-#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
-#include "modules/rtp_rtcp/source/rtp_packet_received.h"
-#include "modules/rtp_rtcp/source/rtp_rtcp_config.h"
-#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
-#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h"
-#include "modules/rtp_rtcp/source/video_rtp_depacketizer_raw.h"
-#include "modules/utility/include/process_thread.h"
-#include "modules/video_coding/deprecated/nack_module.h"
-#include "modules/video_coding/frame_object.h"
-#include "modules/video_coding/h264_sprop_parameter_sets.h"
-#include "modules/video_coding/h264_sps_pps_tracker.h"
-#include "modules/video_coding/packet_buffer.h"
-#include "rtc_base/checks.h"
-#include "rtc_base/location.h"
-#include "rtc_base/logging.h"
-#include "rtc_base/strings/string_builder.h"
-#include "system_wrappers/include/metrics.h"
-#include "system_wrappers/include/ntp_time.h"
-#include "video/receive_statistics_proxy.h"
-
-namespace webrtc {
-
-namespace {
-// TODO(philipel): Change kPacketBufferStartSize back to 32 in M63 see:
-// crbug.com/752886
-constexpr int kPacketBufferStartSize = 512;
-constexpr int kPacketBufferMaxSize = 2048;
-
-int PacketBufferMaxSize(const FieldTrialsView& field_trials) {
- // The group here must be a positive power of 2, in which case that is used as
- // size. All other values shall result in the default value being used.
- const std::string group_name =
- field_trials.Lookup("WebRTC-PacketBufferMaxSize");
- int packet_buffer_max_size = kPacketBufferMaxSize;
- if (!group_name.empty() &&
- (sscanf(group_name.c_str(), "%d", &packet_buffer_max_size) != 1 ||
- packet_buffer_max_size <= 0 ||
- // Verify that the number is a positive power of 2.
- (packet_buffer_max_size & (packet_buffer_max_size - 1)) != 0)) {
- RTC_LOG(LS_WARNING) << "Invalid packet buffer max size: " << group_name;
- packet_buffer_max_size = kPacketBufferMaxSize;
- }
- return packet_buffer_max_size;
-}
-
-std::unique_ptr<RtpRtcp> CreateRtpRtcpModule(
- Clock* clock,
- ReceiveStatistics* receive_statistics,
- Transport* outgoing_transport,
- RtcpRttStats* rtt_stats,
- RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
- RtcpCnameCallback* rtcp_cname_callback,
- bool non_sender_rtt_measurement,
- uint32_t local_ssrc) {
- RtpRtcpInterface::Configuration configuration;
- configuration.clock = clock;
- configuration.audio = false;
- configuration.receiver_only = true;
- configuration.receive_statistics = receive_statistics;
- configuration.outgoing_transport = outgoing_transport;
- configuration.rtt_stats = rtt_stats;
- configuration.rtcp_packet_type_counter_observer =
- rtcp_packet_type_counter_observer;
- configuration.rtcp_cname_callback = rtcp_cname_callback;
- configuration.local_media_ssrc = local_ssrc;
- configuration.non_sender_rtt_measurement = non_sender_rtt_measurement;
-
- std::unique_ptr<RtpRtcp> rtp_rtcp = RtpRtcp::DEPRECATED_Create(configuration);
- rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound);
-
- return rtp_rtcp;
-}
-
-static const int kPacketLogIntervalMs = 10000;
-
-} // namespace
-
-RtpVideoStreamReceiver::RtcpFeedbackBuffer::RtcpFeedbackBuffer(
- KeyFrameRequestSender* key_frame_request_sender,
- NackSender* nack_sender,
- LossNotificationSender* loss_notification_sender)
- : key_frame_request_sender_(key_frame_request_sender),
- nack_sender_(nack_sender),
- loss_notification_sender_(loss_notification_sender),
- request_key_frame_(false) {
- RTC_DCHECK(key_frame_request_sender_);
- RTC_DCHECK(nack_sender_);
- RTC_DCHECK(loss_notification_sender_);
-}
-
-void RtpVideoStreamReceiver::RtcpFeedbackBuffer::RequestKeyFrame() {
- MutexLock lock(&mutex_);
- request_key_frame_ = true;
-}
-
-void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendNack(
- const std::vector<uint16_t>& sequence_numbers,
- bool buffering_allowed) {
- RTC_DCHECK(!sequence_numbers.empty());
- MutexLock lock(&mutex_);
- nack_sequence_numbers_.insert(nack_sequence_numbers_.end(),
- sequence_numbers.cbegin(),
- sequence_numbers.cend());
- if (!buffering_allowed) {
- // Note that while *buffering* is not allowed, *batching* is, meaning that
- // previously buffered messages may be sent along with the current message.
- SendRtcpFeedback(ConsumeRtcpFeedbackLocked());
- }
-}
-
-void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendLossNotification(
- uint16_t last_decoded_seq_num,
- uint16_t last_received_seq_num,
- bool decodability_flag,
- bool buffering_allowed) {
- RTC_DCHECK(buffering_allowed);
- MutexLock lock(&mutex_);
- RTC_DCHECK(!lntf_state_)
- << "SendLossNotification() called twice in a row with no call to "
- "SendBufferedRtcpFeedback() in between.";
- lntf_state_ = absl::make_optional<LossNotificationState>(
- last_decoded_seq_num, last_received_seq_num, decodability_flag);
-}
-
-void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendBufferedRtcpFeedback() {
- SendRtcpFeedback(ConsumeRtcpFeedback());
-}
-
-RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumedRtcpFeedback
-RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumeRtcpFeedback() {
- MutexLock lock(&mutex_);
- return ConsumeRtcpFeedbackLocked();
-}
-
-RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumedRtcpFeedback
-RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumeRtcpFeedbackLocked() {
- ConsumedRtcpFeedback feedback;
- std::swap(feedback.request_key_frame, request_key_frame_);
- std::swap(feedback.nack_sequence_numbers, nack_sequence_numbers_);
- std::swap(feedback.lntf_state, lntf_state_);
- return feedback;
-}
-
-void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendRtcpFeedback(
- ConsumedRtcpFeedback feedback) {
- if (feedback.lntf_state) {
- // If either a NACK or a key frame request is sent, we should buffer
- // the LNTF and wait for them (NACK or key frame request) to trigger
- // the compound feedback message.
- // Otherwise, the LNTF should be sent out immediately.
- const bool buffering_allowed =
- feedback.request_key_frame || !feedback.nack_sequence_numbers.empty();
-
- loss_notification_sender_->SendLossNotification(
- feedback.lntf_state->last_decoded_seq_num,
- feedback.lntf_state->last_received_seq_num,
- feedback.lntf_state->decodability_flag, buffering_allowed);
- }
-
- if (feedback.request_key_frame) {
- key_frame_request_sender_->RequestKeyFrame();
- } else if (!feedback.nack_sequence_numbers.empty()) {
- nack_sender_->SendNack(feedback.nack_sequence_numbers, true);
- }
-}
-
-// DEPRECATED
-RtpVideoStreamReceiver::RtpVideoStreamReceiver(
- Clock* clock,
- Transport* transport,
- RtcpRttStats* rtt_stats,
- PacketRouter* packet_router,
- const VideoReceiveStreamInterface::Config* config,
- ReceiveStatistics* rtp_receive_statistics,
- ReceiveStatisticsProxy* receive_stats_proxy,
- ProcessThread* process_thread,
- NackSender* nack_sender,
- KeyFrameRequestSender* keyframe_request_sender,
- OnCompleteFrameCallback* complete_frame_callback,
- rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
- rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
- const FieldTrialsView* field_trials)
- : RtpVideoStreamReceiver(clock,
- transport,
- rtt_stats,
- packet_router,
- config,
- rtp_receive_statistics,
- receive_stats_proxy,
- receive_stats_proxy,
- process_thread,
- nack_sender,
- keyframe_request_sender,
- complete_frame_callback,
- frame_decryptor,
- frame_transformer,
- field_trials) {}
-
-RtpVideoStreamReceiver::RtpVideoStreamReceiver(
- Clock* clock,
- Transport* transport,
- RtcpRttStats* rtt_stats,
- PacketRouter* packet_router,
- const VideoReceiveStreamInterface::Config* config,
- ReceiveStatistics* rtp_receive_statistics,
- RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
- RtcpCnameCallback* rtcp_cname_callback,
- ProcessThread* process_thread,
- NackSender* nack_sender,
- KeyFrameRequestSender* keyframe_request_sender,
- OnCompleteFrameCallback* complete_frame_callback,
- rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
- rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
- const FieldTrialsView* field_trials)
- : field_trials_(field_trials ? *field_trials : owned_field_trials_),
- clock_(clock),
- config_(*config),
- packet_router_(packet_router),
- process_thread_(process_thread),
- ntp_estimator_(clock),
- rtp_header_extensions_(config_.rtp.extensions),
- forced_playout_delay_max_ms_("max_ms", absl::nullopt),
- forced_playout_delay_min_ms_("min_ms", absl::nullopt),
- rtp_receive_statistics_(rtp_receive_statistics),
- ulpfec_receiver_(UlpfecReceiver::Create(config->rtp.remote_ssrc,
- this,
- config->rtp.extensions)),
- receiving_(false),
- last_packet_log_ms_(-1),
- rtp_rtcp_(CreateRtpRtcpModule(
- clock,
- rtp_receive_statistics_,
- transport,
- rtt_stats,
- rtcp_packet_type_counter_observer,
- rtcp_cname_callback,
- config_.rtp.rtcp_xr.receiver_reference_time_report,
- config_.rtp.local_ssrc)),
- complete_frame_callback_(complete_frame_callback),
- keyframe_request_sender_(keyframe_request_sender),
- // TODO(bugs.webrtc.org/10336): Let `rtcp_feedback_buffer_` communicate
- // directly with `rtp_rtcp_`.
- rtcp_feedback_buffer_(this, nack_sender, this),
- packet_buffer_(kPacketBufferStartSize,
- PacketBufferMaxSize(field_trials_)),
- reference_finder_(std::make_unique<RtpFrameReferenceFinder>()),
- has_received_frame_(false),
- frames_decryptable_(false),
- absolute_capture_time_interpolator_(clock) {
- constexpr bool remb_candidate = true;
- if (packet_router_)
- packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate);
-
- RTC_DCHECK(config_.rtp.rtcp_mode != RtcpMode::kOff)
- << "A stream should not be configured with RTCP disabled. This value is "
- "reserved for internal usage.";
- // TODO(pbos): What's an appropriate local_ssrc for receive-only streams?
- RTC_DCHECK(config_.rtp.local_ssrc != 0);
- RTC_DCHECK(config_.rtp.remote_ssrc != config_.rtp.local_ssrc);
-
- rtp_rtcp_->SetRTCPStatus(config_.rtp.rtcp_mode);
- rtp_rtcp_->SetRemoteSSRC(config_.rtp.remote_ssrc);
-
- static const int kMaxPacketAgeToNack = 450;
- const int max_reordering_threshold = (config_.rtp.nack.rtp_history_ms > 0)
- ? kMaxPacketAgeToNack
- : kDefaultMaxReorderingThreshold;
- rtp_receive_statistics_->SetMaxReorderingThreshold(config_.rtp.remote_ssrc,
- max_reordering_threshold);
- // TODO(nisse): For historic reasons, we applied the above
- // max_reordering_threshold also for RTX stats, which makes little sense since
- // we don't NACK rtx packets. Consider deleting the below block, and rely on
- // the default threshold.
- if (config_.rtp.rtx_ssrc) {
- rtp_receive_statistics_->SetMaxReorderingThreshold(
- config_.rtp.rtx_ssrc, max_reordering_threshold);
- }
- ParseFieldTrial(
- {&forced_playout_delay_max_ms_, &forced_playout_delay_min_ms_},
- field_trials_.Lookup("WebRTC-ForcePlayoutDelay"));
-
- process_thread_->RegisterModule(rtp_rtcp_.get(), RTC_FROM_HERE);
-
- if (config_.rtp.lntf.enabled) {
- loss_notification_controller_ =
- std::make_unique<LossNotificationController>(&rtcp_feedback_buffer_,
- &rtcp_feedback_buffer_);
- }
-
- if (config_.rtp.nack.rtp_history_ms != 0) {
- nack_module_ = std::make_unique<DEPRECATED_NackModule>(
- clock_, &rtcp_feedback_buffer_, &rtcp_feedback_buffer_, field_trials_);
- process_thread_->RegisterModule(nack_module_.get(), RTC_FROM_HERE);
- }
-
- // Only construct the encrypted receiver if frame encryption is enabled.
- if (config_.crypto_options.sframe.require_frame_encryption) {
- buffered_frame_decryptor_ =
- std::make_unique<BufferedFrameDecryptor>(this, this, field_trials_);
- if (frame_decryptor != nullptr) {
- buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor));
- }
- }
-
- if (frame_transformer) {
- frame_transformer_delegate_ =
- rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
- this, std::move(frame_transformer), rtc::Thread::Current(),
- config_.rtp.remote_ssrc);
- frame_transformer_delegate_->Init();
- }
-}
-
-RtpVideoStreamReceiver::~RtpVideoStreamReceiver() {
- RTC_DCHECK(secondary_sinks_.empty());
-
- if (nack_module_) {
- process_thread_->DeRegisterModule(nack_module_.get());
- }
-
- process_thread_->DeRegisterModule(rtp_rtcp_.get());
-
- if (packet_router_)
- packet_router_->RemoveReceiveRtpModule(rtp_rtcp_.get());
- UpdateHistograms();
- if (frame_transformer_delegate_)
- frame_transformer_delegate_->Reset();
-}
-
-void RtpVideoStreamReceiver::AddReceiveCodec(
- uint8_t payload_type,
- VideoCodecType codec_type,
- const std::map<std::string, std::string>& codec_params,
- bool raw_payload) {
- if (codec_params.count(cricket::kH264FmtpSpsPpsIdrInKeyframe) ||
- field_trials_.IsEnabled("WebRTC-SpsPpsIdrIsH264Keyframe")) {
- MutexLock lock(&packet_buffer_lock_);
- packet_buffer_.ForceSpsPpsIdrIsH264Keyframe();
- }
- payload_type_map_.emplace(
- payload_type, raw_payload ? std::make_unique<VideoRtpDepacketizerRaw>()
- : CreateVideoRtpDepacketizer(codec_type));
- pt_codec_params_.emplace(payload_type, codec_params);
-}
-
-absl::optional<Syncable::Info> RtpVideoStreamReceiver::GetSyncInfo() const {
- Syncable::Info info;
- if (rtp_rtcp_->RemoteNTP(&info.capture_time_ntp_secs,
- &info.capture_time_ntp_frac,
- /*rtcp_arrival_time_secs=*/nullptr,
- /*rtcp_arrival_time_frac=*/nullptr,
- &info.capture_time_source_clock) != 0) {
- return absl::nullopt;
- }
- {
- MutexLock lock(&sync_info_lock_);
- if (!last_received_rtp_timestamp_ || !last_received_rtp_system_time_) {
- return absl::nullopt;
- }
- info.latest_received_capture_timestamp = *last_received_rtp_timestamp_;
- info.latest_receive_time_ms = last_received_rtp_system_time_->ms();
- }
-
- // Leaves info.current_delay_ms uninitialized.
- return info;
-}
-
-RtpVideoStreamReceiver::ParseGenericDependenciesResult
-RtpVideoStreamReceiver::ParseGenericDependenciesExtension(
- const RtpPacketReceived& rtp_packet,
- RTPVideoHeader* video_header) {
- if (rtp_packet.HasExtension<RtpDependencyDescriptorExtension>()) {
- webrtc::DependencyDescriptor dependency_descriptor;
- if (!rtp_packet.GetExtension<RtpDependencyDescriptorExtension>(
- video_structure_.get(), &dependency_descriptor)) {
- // Descriptor is there, but failed to parse. Either it is invalid,
- // or too old packet (after relevant video_structure_ changed),
- // or too new packet (before relevant video_structure_ arrived).
- // Drop such packet to be on the safe side.
- // TODO(bugs.webrtc.org/10342): Stash too new packet.
- RTC_LOG(LS_WARNING) << "ssrc: " << rtp_packet.Ssrc()
- << " Failed to parse dependency descriptor.";
- return kDropPacket;
- }
- if (dependency_descriptor.attached_structure != nullptr &&
- !dependency_descriptor.first_packet_in_frame) {
- RTC_LOG(LS_WARNING) << "ssrc: " << rtp_packet.Ssrc()
- << "Invalid dependency descriptor: structure "
- "attached to non first packet of a frame.";
- return kDropPacket;
- }
- video_header->is_first_packet_in_frame =
- dependency_descriptor.first_packet_in_frame;
- video_header->is_last_packet_in_frame =
- dependency_descriptor.last_packet_in_frame;
-
- int64_t frame_id =
- frame_id_unwrapper_.Unwrap(dependency_descriptor.frame_number);
- auto& generic_descriptor_info = video_header->generic.emplace();
- generic_descriptor_info.frame_id = frame_id;
- generic_descriptor_info.spatial_index =
- dependency_descriptor.frame_dependencies.spatial_id;
- generic_descriptor_info.temporal_index =
- dependency_descriptor.frame_dependencies.temporal_id;
- for (int fdiff : dependency_descriptor.frame_dependencies.frame_diffs) {
- generic_descriptor_info.dependencies.push_back(frame_id - fdiff);
- }
- generic_descriptor_info.decode_target_indications =
- dependency_descriptor.frame_dependencies.decode_target_indications;
- if (dependency_descriptor.resolution) {
- video_header->width = dependency_descriptor.resolution->Width();
- video_header->height = dependency_descriptor.resolution->Height();
- }
-
- // FrameDependencyStructure is sent in dependency descriptor of the first
- // packet of a key frame and required for parsed dependency descriptor in
- // all the following packets until next key frame.
- // Save it if there is a (potentially) new structure.
- if (dependency_descriptor.attached_structure) {
- RTC_DCHECK(dependency_descriptor.first_packet_in_frame);
- if (video_structure_frame_id_ > frame_id) {
- RTC_LOG(LS_WARNING)
- << "Arrived key frame with id " << frame_id << " and structure id "
- << dependency_descriptor.attached_structure->structure_id
- << " is older than the latest received key frame with id "
- << *video_structure_frame_id_ << " and structure id "
- << video_structure_->structure_id;
- return kDropPacket;
- }
- video_structure_ = std::move(dependency_descriptor.attached_structure);
- video_structure_frame_id_ = frame_id;
- video_header->frame_type = VideoFrameType::kVideoFrameKey;
- } else {
- video_header->frame_type = VideoFrameType::kVideoFrameDelta;
- }
- return kHasGenericDescriptor;
- }
-
- RtpGenericFrameDescriptor generic_frame_descriptor;
- if (!rtp_packet.GetExtension<RtpGenericFrameDescriptorExtension00>(
- &generic_frame_descriptor)) {
- return kNoGenericDescriptor;
- }
-
- video_header->is_first_packet_in_frame =
- generic_frame_descriptor.FirstPacketInSubFrame();
- video_header->is_last_packet_in_frame =
- generic_frame_descriptor.LastPacketInSubFrame();
-
- if (generic_frame_descriptor.FirstPacketInSubFrame()) {
- video_header->frame_type =
- generic_frame_descriptor.FrameDependenciesDiffs().empty()
- ? VideoFrameType::kVideoFrameKey
- : VideoFrameType::kVideoFrameDelta;
-
- auto& generic_descriptor_info = video_header->generic.emplace();
- int64_t frame_id =
- frame_id_unwrapper_.Unwrap(generic_frame_descriptor.FrameId());
- generic_descriptor_info.frame_id = frame_id;
- generic_descriptor_info.spatial_index =
- generic_frame_descriptor.SpatialLayer();
- generic_descriptor_info.temporal_index =
- generic_frame_descriptor.TemporalLayer();
- for (uint16_t fdiff : generic_frame_descriptor.FrameDependenciesDiffs()) {
- generic_descriptor_info.dependencies.push_back(frame_id - fdiff);
- }
- }
- video_header->width = generic_frame_descriptor.Width();
- video_header->height = generic_frame_descriptor.Height();
- return kHasGenericDescriptor;
-}
-
-void RtpVideoStreamReceiver::OnReceivedPayloadData(
- rtc::CopyOnWriteBuffer codec_payload,
- const RtpPacketReceived& rtp_packet,
- const RTPVideoHeader& video) {
- RTC_DCHECK_RUN_ON(&worker_task_checker_);
-
- auto packet =
- std::make_unique<video_coding::PacketBuffer::Packet>(rtp_packet, video);
-
- RTPVideoHeader& video_header = packet->video_header;
- video_header.rotation = kVideoRotation_0;
- video_header.content_type = VideoContentType::UNSPECIFIED;
- video_header.video_timing.flags = VideoSendTiming::kInvalid;
- video_header.is_last_packet_in_frame |= rtp_packet.Marker();
-
- if (const auto* vp9_header =
- absl::get_if<RTPVideoHeaderVP9>(&video_header.video_type_header)) {
- video_header.is_last_packet_in_frame |= vp9_header->end_of_frame;
- video_header.is_first_packet_in_frame |= vp9_header->beginning_of_frame;
- }
-
- rtp_packet.GetExtension<VideoOrientation>(&video_header.rotation);
- rtp_packet.GetExtension<VideoContentTypeExtension>(
- &video_header.content_type);
- rtp_packet.GetExtension<VideoTimingExtension>(&video_header.video_timing);
- if (forced_playout_delay_max_ms_ && forced_playout_delay_min_ms_) {
- video_header.playout_delay.max_ms = *forced_playout_delay_max_ms_;
- video_header.playout_delay.min_ms = *forced_playout_delay_min_ms_;
- } else {
- rtp_packet.GetExtension<PlayoutDelayLimits>(&video_header.playout_delay);
- }
-
- ParseGenericDependenciesResult generic_descriptor_state =
- ParseGenericDependenciesExtension(rtp_packet, &video_header);
-
- if (!rtp_packet.recovered()) {
- UpdatePacketReceiveTimestamps(
- rtp_packet, video_header.frame_type == VideoFrameType::kVideoFrameKey);
- }
-
- if (generic_descriptor_state == kDropPacket)
- return;
-
- // Color space should only be transmitted in the last packet of a frame,
- // therefore, neglect it otherwise so that last_color_space_ is not reset by
- // mistake.
- if (video_header.is_last_packet_in_frame) {
- video_header.color_space = rtp_packet.GetExtension<ColorSpaceExtension>();
- if (video_header.color_space ||
- video_header.frame_type == VideoFrameType::kVideoFrameKey) {
- // Store color space since it's only transmitted when changed or for key
- // frames. Color space will be cleared if a key frame is transmitted
- // without color space information.
- last_color_space_ = video_header.color_space;
- } else if (last_color_space_) {
- video_header.color_space = last_color_space_;
- }
- }
- video_header.video_frame_tracking_id =
- rtp_packet.GetExtension<VideoFrameTrackingIdExtension>();
-
- if (loss_notification_controller_) {
- if (rtp_packet.recovered()) {
- // TODO(bugs.webrtc.org/10336): Implement support for reordering.
- RTC_LOG(LS_INFO)
- << "LossNotificationController does not support reordering.";
- } else if (generic_descriptor_state == kNoGenericDescriptor) {
- RTC_LOG(LS_WARNING) << "LossNotificationController requires generic "
- "frame descriptor, but it is missing.";
- } else {
- if (video_header.is_first_packet_in_frame) {
- RTC_DCHECK(video_header.generic);
- LossNotificationController::FrameDetails frame;
- frame.is_keyframe =
- video_header.frame_type == VideoFrameType::kVideoFrameKey;
- frame.frame_id = video_header.generic->frame_id;
- frame.frame_dependencies = video_header.generic->dependencies;
- loss_notification_controller_->OnReceivedPacket(
- rtp_packet.SequenceNumber(), &frame);
- } else {
- loss_notification_controller_->OnReceivedPacket(
- rtp_packet.SequenceNumber(), nullptr);
- }
- }
- }
-
- if (nack_module_) {
- const bool is_keyframe =
- video_header.is_first_packet_in_frame &&
- video_header.frame_type == VideoFrameType::kVideoFrameKey;
-
- packet->times_nacked = nack_module_->OnReceivedPacket(
- rtp_packet.SequenceNumber(), is_keyframe, rtp_packet.recovered());
- } else {
- packet->times_nacked = -1;
- }
-
- if (codec_payload.size() == 0) {
- NotifyReceiverOfEmptyPacket(packet->seq_num);
- rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
- return;
- }
-
- if (packet->codec() == kVideoCodecH264) {
- // Only when we start to receive packets will we know what payload type
- // that will be used. When we know the payload type insert the correct
- // sps/pps into the tracker.
- if (packet->payload_type != last_payload_type_) {
- last_payload_type_ = packet->payload_type;
- InsertSpsPpsIntoTracker(packet->payload_type);
- }
-
- video_coding::H264SpsPpsTracker::FixedBitstream fixed =
- tracker_.CopyAndFixBitstream(
- rtc::MakeArrayView(codec_payload.cdata(), codec_payload.size()),
- &packet->video_header);
-
- switch (fixed.action) {
- case video_coding::H264SpsPpsTracker::kRequestKeyframe:
- rtcp_feedback_buffer_.RequestKeyFrame();
- rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
- [[fallthrough]];
- case video_coding::H264SpsPpsTracker::kDrop:
- return;
- case video_coding::H264SpsPpsTracker::kInsert:
- packet->video_payload = std::move(fixed.bitstream);
- break;
- }
-
- } else {
- packet->video_payload = std::move(codec_payload);
- }
-
- rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
- frame_counter_.Add(packet->timestamp);
- video_coding::PacketBuffer::InsertResult insert_result;
- {
- MutexLock lock(&packet_buffer_lock_);
- int64_t unwrapped_rtp_seq_num =
- rtp_seq_num_unwrapper_.Unwrap(rtp_packet.SequenceNumber());
- auto& packet_info =
- packet_infos_
- .emplace(
- unwrapped_rtp_seq_num,
- RtpPacketInfo(
- rtp_packet.Ssrc(), rtp_packet.Csrcs(),
- rtp_packet.Timestamp(),
- /*audio_level=*/absl::nullopt,
- rtp_packet.GetExtension<AbsoluteCaptureTimeExtension>(),
- /*receive_time_ms=*/clock_->TimeInMilliseconds()))
- .first->second;
-
- // Try to extrapolate absolute capture time if it is missing.
- absl::optional<AbsoluteCaptureTime> absolute_capture_time =
- absolute_capture_time_interpolator_.OnReceivePacket(
- AbsoluteCaptureTimeInterpolator::GetSource(packet_info.ssrc(),
- packet_info.csrcs()),
- packet_info.rtp_timestamp(),
- // Assume frequency is the same one for all video frames.
- kVideoPayloadTypeFrequency, packet_info.absolute_capture_time());
- packet_info.set_absolute_capture_time(absolute_capture_time);
-
- if (absolute_capture_time.has_value()) {
- packet_info.set_local_capture_clock_offset(
- capture_clock_offset_updater_.AdjustEstimatedCaptureClockOffset(
- absolute_capture_time->estimated_capture_clock_offset));
- }
-
- insert_result = packet_buffer_.InsertPacket(std::move(packet));
- }
- OnInsertedPacket(std::move(insert_result));
-}
-
-void RtpVideoStreamReceiver::OnRecoveredPacket(const uint8_t* rtp_packet,
- size_t rtp_packet_length) {
- RtpPacketReceived packet;
- if (!packet.Parse(rtp_packet, rtp_packet_length))
- return;
- if (packet.PayloadType() == config_.rtp.red_payload_type) {
- RTC_LOG(LS_WARNING) << "Discarding recovered packet with RED encapsulation";
- return;
- }
-
- packet.IdentifyExtensions(rtp_header_extensions_);
- packet.set_payload_type_frequency(kVideoPayloadTypeFrequency);
- // TODO(nisse): UlpfecReceiverImpl::ProcessReceivedFec passes both
- // original (decapsulated) media packets and recovered packets to
- // this callback. We need a way to distinguish, for setting
- // packet.recovered() correctly. Ideally, move RED decapsulation out
- // of the Ulpfec implementation.
-
- ReceivePacket(packet);
-}
-
-// This method handles both regular RTP packets and packets recovered
-// via FlexFEC.
-void RtpVideoStreamReceiver::OnRtpPacket(const RtpPacketReceived& packet) {
- RTC_DCHECK_RUN_ON(&worker_task_checker_);
-
- if (!receiving_) {
- return;
- }
-
- ReceivePacket(packet);
-
- // Update receive statistics after ReceivePacket.
- // Receive statistics will be reset if the payload type changes (make sure
- // that the first packet is included in the stats).
- if (!packet.recovered()) {
- rtp_receive_statistics_->OnRtpPacket(packet);
- }
-
- for (RtpPacketSinkInterface* secondary_sink : secondary_sinks_) {
- secondary_sink->OnRtpPacket(packet);
- }
-}
-
-void RtpVideoStreamReceiver::RequestKeyFrame() {
- // TODO(bugs.webrtc.org/10336): Allow the sender to ignore key frame requests
- // issued by anything other than the LossNotificationController if it (the
- // sender) is relying on LNTF alone.
- if (keyframe_request_sender_) {
- keyframe_request_sender_->RequestKeyFrame();
- } else {
- rtp_rtcp_->SendPictureLossIndication();
- }
-}
-
-void RtpVideoStreamReceiver::SendLossNotification(
- uint16_t last_decoded_seq_num,
- uint16_t last_received_seq_num,
- bool decodability_flag,
- bool buffering_allowed) {
- RTC_DCHECK(config_.rtp.lntf.enabled);
- rtp_rtcp_->SendLossNotification(last_decoded_seq_num, last_received_seq_num,
- decodability_flag, buffering_allowed);
-}
-
-bool RtpVideoStreamReceiver::IsUlpfecEnabled() const {
- return config_.rtp.ulpfec_payload_type != -1;
-}
-
-bool RtpVideoStreamReceiver::IsRetransmissionsEnabled() const {
- return config_.rtp.nack.rtp_history_ms > 0;
-}
-
-void RtpVideoStreamReceiver::RequestPacketRetransmit(
- const std::vector<uint16_t>& sequence_numbers) {
- rtp_rtcp_->SendNack(sequence_numbers);
-}
-
-bool RtpVideoStreamReceiver::IsDecryptable() const {
- return frames_decryptable_.load();
-}
-
-void RtpVideoStreamReceiver::OnInsertedPacket(
- video_coding::PacketBuffer::InsertResult result) {
- std::vector<std::unique_ptr<RtpFrameObject>> assembled_frames;
- {
- MutexLock lock(&packet_buffer_lock_);
- video_coding::PacketBuffer::Packet* first_packet = nullptr;
- int max_nack_count;
- int64_t min_recv_time;
- int64_t max_recv_time;
- std::vector<rtc::ArrayView<const uint8_t>> payloads;
- RtpPacketInfos::vector_type packet_infos;
-
- bool frame_boundary = true;
- for (auto& packet : result.packets) {
- // PacketBuffer promisses frame boundaries are correctly set on each
- // packet. Document that assumption with the DCHECKs.
- RTC_DCHECK_EQ(frame_boundary, packet->is_first_packet_in_frame());
- int64_t unwrapped_rtp_seq_num =
- rtp_seq_num_unwrapper_.Unwrap(packet->seq_num);
- RTC_DCHECK(packet_infos_.count(unwrapped_rtp_seq_num) > 0);
- RtpPacketInfo& packet_info = packet_infos_[unwrapped_rtp_seq_num];
- if (packet->is_first_packet_in_frame()) {
- first_packet = packet.get();
- max_nack_count = packet->times_nacked;
- min_recv_time = packet_info.receive_time().ms();
- max_recv_time = packet_info.receive_time().ms();
- } else {
- max_nack_count = std::max(max_nack_count, packet->times_nacked);
- min_recv_time =
- std::min(min_recv_time, packet_info.receive_time().ms());
- max_recv_time =
- std::max(max_recv_time, packet_info.receive_time().ms());
- }
- payloads.emplace_back(packet->video_payload);
- packet_infos.push_back(packet_info);
-
- frame_boundary = packet->is_last_packet_in_frame();
- if (packet->is_last_packet_in_frame()) {
- auto depacketizer_it =
- payload_type_map_.find(first_packet->payload_type);
- RTC_CHECK(depacketizer_it != payload_type_map_.end());
-
- rtc::scoped_refptr<EncodedImageBuffer> bitstream =
- depacketizer_it->second->AssembleFrame(payloads);
- if (!bitstream) {
- // Failed to assemble a frame. Discard and continue.
- continue;
- }
-
- const video_coding::PacketBuffer::Packet& last_packet = *packet;
- assembled_frames.push_back(std::make_unique<RtpFrameObject>(
- first_packet->seq_num, //
- last_packet.seq_num, //
- last_packet.marker_bit, //
- max_nack_count, //
- min_recv_time, //
- max_recv_time, //
- first_packet->timestamp, //
- ntp_estimator_.Estimate(first_packet->timestamp), //
- last_packet.video_header.video_timing, //
- first_packet->payload_type, //
- first_packet->codec(), //
- last_packet.video_header.rotation, //
- last_packet.video_header.content_type, //
- first_packet->video_header, //
- last_packet.video_header.color_space, //
- RtpPacketInfos(std::move(packet_infos)), //
- std::move(bitstream)));
- payloads.clear();
- packet_infos.clear();
- }
- }
- RTC_DCHECK(frame_boundary);
-
- if (result.buffer_cleared) {
- packet_infos_.clear();
- }
- } // packet_buffer_lock_
-
- if (result.buffer_cleared) {
- {
- MutexLock lock(&sync_info_lock_);
- last_received_rtp_system_time_.reset();
- last_received_keyframe_rtp_system_time_.reset();
- last_received_keyframe_rtp_timestamp_.reset();
- }
- RequestKeyFrame();
- }
-
- for (auto& frame : assembled_frames) {
- OnAssembledFrame(std::move(frame));
- }
-}
-
-void RtpVideoStreamReceiver::OnAssembledFrame(
- std::unique_ptr<RtpFrameObject> frame) {
- RTC_DCHECK_RUN_ON(&network_tc_);
- RTC_DCHECK(frame);
-
- const absl::optional<RTPVideoHeader::GenericDescriptorInfo>& descriptor =
- frame->GetRtpVideoHeader().generic;
-
- if (loss_notification_controller_ && descriptor) {
- loss_notification_controller_->OnAssembledFrame(
- frame->first_seq_num(), descriptor->frame_id,
- absl::c_linear_search(descriptor->decode_target_indications,
- DecodeTargetIndication::kDiscardable),
- descriptor->dependencies);
- }
-
- // If frames arrive before a key frame, they would not be decodable.
- // In that case, request a key frame ASAP.
- if (!has_received_frame_) {
- if (frame->FrameType() != VideoFrameType::kVideoFrameKey) {
- // `loss_notification_controller_`, if present, would have already
- // requested a key frame when the first packet for the non-key frame
- // had arrived, so no need to replicate the request.
- if (!loss_notification_controller_) {
- RequestKeyFrame();
- }
- }
- has_received_frame_ = true;
- }
-
- MutexLock lock(&reference_finder_lock_);
- // Reset `reference_finder_` if `frame` is new and the codec have changed.
- if (current_codec_) {
- bool frame_is_newer =
- AheadOf(frame->Timestamp(), last_assembled_frame_rtp_timestamp_);
-
- if (frame->codec_type() != current_codec_) {
- if (frame_is_newer) {
- // When we reset the `reference_finder_` we don't want new picture ids
- // to overlap with old picture ids. To ensure that doesn't happen we
- // start from the `last_completed_picture_id_` and add an offset in
- // case of reordering.
- reference_finder_ = std::make_unique<RtpFrameReferenceFinder>(
- last_completed_picture_id_ + std::numeric_limits<uint16_t>::max());
- current_codec_ = frame->codec_type();
- } else {
- // Old frame from before the codec switch, discard it.
- return;
- }
- }
-
- if (frame_is_newer) {
- last_assembled_frame_rtp_timestamp_ = frame->Timestamp();
- }
- } else {
- current_codec_ = frame->codec_type();
- last_assembled_frame_rtp_timestamp_ = frame->Timestamp();
- }
-
- if (buffered_frame_decryptor_ != nullptr) {
- buffered_frame_decryptor_->ManageEncryptedFrame(std::move(frame));
- } else if (frame_transformer_delegate_) {
- frame_transformer_delegate_->TransformFrame(std::move(frame));
- } else {
- OnCompleteFrames(reference_finder_->ManageFrame(std::move(frame)));
- }
-}
-
-void RtpVideoStreamReceiver::OnCompleteFrames(
- RtpFrameReferenceFinder::ReturnVector frames) {
- {
- MutexLock lock(&last_seq_num_mutex_);
- for (const auto& frame : frames) {
- RtpFrameObject* rtp_frame = static_cast<RtpFrameObject*>(frame.get());
- last_seq_num_for_pic_id_[rtp_frame->Id()] = rtp_frame->last_seq_num();
- }
- }
- for (auto& frame : frames) {
- last_completed_picture_id_ =
- std::max(last_completed_picture_id_, frame->Id());
- complete_frame_callback_->OnCompleteFrame(std::move(frame));
- }
-}
-
-void RtpVideoStreamReceiver::OnDecryptedFrame(
- std::unique_ptr<RtpFrameObject> frame) {
- MutexLock lock(&reference_finder_lock_);
- OnCompleteFrames(reference_finder_->ManageFrame(std::move(frame)));
-}
-
-void RtpVideoStreamReceiver::OnDecryptionStatusChange(
- FrameDecryptorInterface::Status status) {
- frames_decryptable_.store(
- (status == FrameDecryptorInterface::Status::kOk) ||
- (status == FrameDecryptorInterface::Status::kRecoverable));
-}
-
-void RtpVideoStreamReceiver::SetFrameDecryptor(
- rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) {
- RTC_DCHECK_RUN_ON(&network_tc_);
- if (buffered_frame_decryptor_ == nullptr) {
- buffered_frame_decryptor_ =
- std::make_unique<BufferedFrameDecryptor>(this, this, field_trials_);
- }
- buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor));
-}
-
-void RtpVideoStreamReceiver::SetDepacketizerToDecoderFrameTransformer(
- rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {
- RTC_DCHECK_RUN_ON(&network_tc_);
- frame_transformer_delegate_ =
- rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
- this, std::move(frame_transformer), rtc::Thread::Current(),
- config_.rtp.remote_ssrc);
- frame_transformer_delegate_->Init();
-}
-
-void RtpVideoStreamReceiver::UpdateRtt(int64_t max_rtt_ms) {
- if (nack_module_)
- nack_module_->UpdateRtt(max_rtt_ms);
-}
-
-absl::optional<int64_t> RtpVideoStreamReceiver::LastReceivedPacketMs() const {
- MutexLock lock(&sync_info_lock_);
- if (last_received_rtp_system_time_) {
- return absl::optional<int64_t>(last_received_rtp_system_time_->ms());
- }
- return absl::nullopt;
-}
-
-absl::optional<int64_t> RtpVideoStreamReceiver::LastReceivedKeyframePacketMs()
- const {
- MutexLock lock(&sync_info_lock_);
- if (last_received_keyframe_rtp_system_time_) {
- return absl::optional<int64_t>(
- last_received_keyframe_rtp_system_time_->ms());
- }
- return absl::nullopt;
-}
-
-void RtpVideoStreamReceiver::AddSecondarySink(RtpPacketSinkInterface* sink) {
- RTC_DCHECK_RUN_ON(&worker_task_checker_);
- RTC_DCHECK(!absl::c_linear_search(secondary_sinks_, sink));
- secondary_sinks_.push_back(sink);
-}
-
-void RtpVideoStreamReceiver::RemoveSecondarySink(
- const RtpPacketSinkInterface* sink) {
- RTC_DCHECK_RUN_ON(&worker_task_checker_);
- auto it = absl::c_find(secondary_sinks_, sink);
- if (it == secondary_sinks_.end()) {
- // We might be rolling-back a call whose setup failed mid-way. In such a
- // case, it's simpler to remove "everything" rather than remember what
- // has already been added.
- RTC_LOG(LS_WARNING) << "Removal of unknown sink.";
- return;
- }
- secondary_sinks_.erase(it);
-}
-
-void RtpVideoStreamReceiver::ManageFrame(
- std::unique_ptr<RtpFrameObject> frame) {
- MutexLock lock(&reference_finder_lock_);
- OnCompleteFrames(reference_finder_->ManageFrame(std::move(frame)));
-}
-
-void RtpVideoStreamReceiver::ReceivePacket(const RtpPacketReceived& packet) {
- if (packet.payload_size() == 0) {
- // Padding or keep-alive packet.
- // TODO(nisse): Could drop empty packets earlier, but need to figure out how
- // they should be counted in stats.
- NotifyReceiverOfEmptyPacket(packet.SequenceNumber());
- return;
- }
- if (packet.PayloadType() == config_.rtp.red_payload_type) {
- ParseAndHandleEncapsulatingHeader(packet);
- return;
- }
-
- const auto type_it = payload_type_map_.find(packet.PayloadType());
- if (type_it == payload_type_map_.end()) {
- return;
- }
- absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed_payload =
- type_it->second->Parse(packet.PayloadBuffer());
- if (parsed_payload == absl::nullopt) {
- RTC_LOG(LS_WARNING) << "Failed parsing payload.";
- return;
- }
-
- OnReceivedPayloadData(std::move(parsed_payload->video_payload), packet,
- parsed_payload->video_header);
-}
-
-void RtpVideoStreamReceiver::ParseAndHandleEncapsulatingHeader(
- const RtpPacketReceived& packet) {
- RTC_DCHECK_RUN_ON(&worker_task_checker_);
- if (packet.PayloadType() == config_.rtp.red_payload_type &&
- packet.payload_size() > 0) {
- if (packet.payload()[0] == config_.rtp.ulpfec_payload_type) {
- // Notify video_receiver about received FEC packets to avoid NACKing these
- // packets.
- NotifyReceiverOfEmptyPacket(packet.SequenceNumber());
- }
- if (!ulpfec_receiver_->AddReceivedRedPacket(
- packet, config_.rtp.ulpfec_payload_type)) {
- return;
- }
- ulpfec_receiver_->ProcessReceivedFec();
- }
-}
-
-// In the case of a video stream without picture ids and no rtx the
-// RtpFrameReferenceFinder will need to know about padding to
-// correctly calculate frame references.
-void RtpVideoStreamReceiver::NotifyReceiverOfEmptyPacket(uint16_t seq_num) {
- {
- MutexLock lock(&reference_finder_lock_);
- OnCompleteFrames(reference_finder_->PaddingReceived(seq_num));
- }
-
- video_coding::PacketBuffer::InsertResult insert_result;
- {
- MutexLock lock(&packet_buffer_lock_);
- insert_result = packet_buffer_.InsertPadding(seq_num);
- }
- OnInsertedPacket(std::move(insert_result));
-
- if (nack_module_) {
- nack_module_->OnReceivedPacket(seq_num, /* is_keyframe = */ false,
- /* is _recovered = */ false);
- }
- if (loss_notification_controller_) {
- // TODO(bugs.webrtc.org/10336): Handle empty packets.
- RTC_LOG(LS_WARNING)
- << "LossNotificationController does not expect empty packets.";
- }
-}
-
-bool RtpVideoStreamReceiver::DeliverRtcp(const uint8_t* rtcp_packet,
- size_t rtcp_packet_length) {
- RTC_DCHECK_RUN_ON(&worker_task_checker_);
-
- if (!receiving_) {
- return false;
- }
-
- rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length);
-
- int64_t rtt = 0;
- rtp_rtcp_->RTT(config_.rtp.remote_ssrc, &rtt, nullptr, nullptr, nullptr);
- if (rtt == 0) {
- // Waiting for valid rtt.
- return true;
- }
- uint32_t ntp_secs = 0;
- uint32_t ntp_frac = 0;
- uint32_t rtp_timestamp = 0;
- uint32_t received_ntp_secs = 0;
- uint32_t received_ntp_frac = 0;
- if (rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, &received_ntp_secs,
- &received_ntp_frac, &rtp_timestamp) != 0) {
- // Waiting for RTCP.
- return true;
- }
- NtpTime received_ntp(received_ntp_secs, received_ntp_frac);
- int64_t time_since_received =
- clock_->CurrentNtpInMilliseconds() - received_ntp.ToMs();
- // Don't use old SRs to estimate time.
- if (time_since_received <= 1) {
- ntp_estimator_.UpdateRtcpTimestamp(
- TimeDelta::Millis(rtt), NtpTime(ntp_secs, ntp_frac), rtp_timestamp);
- absl::optional<int64_t> remote_to_local_clock_offset =
- ntp_estimator_.EstimateRemoteToLocalClockOffset();
- if (remote_to_local_clock_offset.has_value()) {
- capture_clock_offset_updater_.SetRemoteToLocalClockOffset(
- *remote_to_local_clock_offset);
- }
- }
-
- return true;
-}
-
-void RtpVideoStreamReceiver::FrameContinuous(int64_t picture_id) {
- if (!nack_module_)
- return;
-
- int seq_num = -1;
- {
- MutexLock lock(&last_seq_num_mutex_);
- auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id);
- if (seq_num_it != last_seq_num_for_pic_id_.end())
- seq_num = seq_num_it->second;
- }
- if (seq_num != -1)
- nack_module_->ClearUpTo(seq_num);
-}
-
-void RtpVideoStreamReceiver::FrameDecoded(int64_t picture_id) {
- int seq_num = -1;
- {
- MutexLock lock(&last_seq_num_mutex_);
- auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id);
- if (seq_num_it != last_seq_num_for_pic_id_.end()) {
- seq_num = seq_num_it->second;
- last_seq_num_for_pic_id_.erase(last_seq_num_for_pic_id_.begin(),
- ++seq_num_it);
- }
- }
- if (seq_num != -1) {
- {
- MutexLock lock(&packet_buffer_lock_);
- packet_buffer_.ClearTo(seq_num);
- int64_t unwrapped_rtp_seq_num = rtp_seq_num_unwrapper_.Unwrap(seq_num);
- packet_infos_.erase(packet_infos_.begin(),
- packet_infos_.upper_bound(unwrapped_rtp_seq_num));
- }
- MutexLock lock(&reference_finder_lock_);
- reference_finder_->ClearTo(seq_num);
- }
-}
-
-void RtpVideoStreamReceiver::SignalNetworkState(NetworkState state) {
- rtp_rtcp_->SetRTCPStatus(state == kNetworkUp ? config_.rtp.rtcp_mode
- : RtcpMode::kOff);
-}
-
-void RtpVideoStreamReceiver::StartReceive() {
- RTC_DCHECK_RUN_ON(&worker_task_checker_);
- receiving_ = true;
-}
-
-void RtpVideoStreamReceiver::StopReceive() {
- RTC_DCHECK_RUN_ON(&worker_task_checker_);
- receiving_ = false;
-}
-
-void RtpVideoStreamReceiver::UpdateHistograms() {
- FecPacketCounter counter = ulpfec_receiver_->GetPacketCounter();
- if (counter.first_packet_time_ms == -1)
- return;
-
- int64_t elapsed_sec =
- (clock_->TimeInMilliseconds() - counter.first_packet_time_ms) / 1000;
- if (elapsed_sec < metrics::kMinRunTimeInSeconds)
- return;
-
- if (counter.num_packets > 0) {
- RTC_HISTOGRAM_PERCENTAGE(
- "WebRTC.Video.ReceivedFecPacketsInPercent",
- static_cast<int>(counter.num_fec_packets * 100 / counter.num_packets));
- }
- if (counter.num_fec_packets > 0) {
- RTC_HISTOGRAM_PERCENTAGE("WebRTC.Video.RecoveredMediaPacketsInPercentOfFec",
- static_cast<int>(counter.num_recovered_packets *
- 100 / counter.num_fec_packets));
- }
- if (config_.rtp.ulpfec_payload_type != -1) {
- RTC_HISTOGRAM_COUNTS_10000(
- "WebRTC.Video.FecBitrateReceivedInKbps",
- static_cast<int>(counter.num_bytes * 8 / elapsed_sec / 1000));
- }
-}
-
-void RtpVideoStreamReceiver::InsertSpsPpsIntoTracker(uint8_t payload_type) {
- auto codec_params_it = pt_codec_params_.find(payload_type);
- if (codec_params_it == pt_codec_params_.end())
- return;
-
- RTC_LOG(LS_INFO) << "Found out of band supplied codec parameters for"
- " payload type: "
- << static_cast<int>(payload_type);
-
- H264SpropParameterSets sprop_decoder;
- auto sprop_base64_it =
- codec_params_it->second.find(cricket::kH264FmtpSpropParameterSets);
-
- if (sprop_base64_it == codec_params_it->second.end())
- return;
-
- if (!sprop_decoder.DecodeSprop(sprop_base64_it->second.c_str()))
- return;
-
- tracker_.InsertSpsPpsNalus(sprop_decoder.sps_nalu(),
- sprop_decoder.pps_nalu());
-}
-
-void RtpVideoStreamReceiver::UpdatePacketReceiveTimestamps(
- const RtpPacketReceived& packet,
- bool is_keyframe) {
- Timestamp now = clock_->CurrentTime();
- {
- MutexLock lock(&sync_info_lock_);
- if (is_keyframe ||
- last_received_keyframe_rtp_timestamp_ == packet.Timestamp()) {
- last_received_keyframe_rtp_timestamp_ = packet.Timestamp();
- last_received_keyframe_rtp_system_time_ = now;
- }
- last_received_rtp_system_time_ = now;
- last_received_rtp_timestamp_ = packet.Timestamp();
- }
-
- // Periodically log the RTP header of incoming packets.
- if (now.ms() - last_packet_log_ms_ > kPacketLogIntervalMs) {
- rtc::StringBuilder ss;
- ss << "Packet received on SSRC: " << packet.Ssrc()
- << " with payload type: " << static_cast<int>(packet.PayloadType())
- << ", timestamp: " << packet.Timestamp()
- << ", sequence number: " << packet.SequenceNumber()
- << ", arrival time: " << ToString(packet.arrival_time());
- int32_t time_offset;
- if (packet.GetExtension<TransmissionOffset>(&time_offset)) {
- ss << ", toffset: " << time_offset;
- }
- uint32_t send_time;
- if (packet.GetExtension<AbsoluteSendTime>(&send_time)) {
- ss << ", abs send time: " << send_time;
- }
- RTC_LOG(LS_INFO) << ss.str();
- last_packet_log_ms_ = now.ms();
- }
-}
-
-} // namespace webrtc
diff --git a/video/rtp_video_stream_receiver.h b/video/rtp_video_stream_receiver.h
deleted file mode 100644
index 4578857..0000000
--- a/video/rtp_video_stream_receiver.h
+++ /dev/null
@@ -1,436 +0,0 @@
-/*
- * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_
-#define VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_
-
-#include <atomic>
-#include <list>
-#include <map>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "absl/types/optional.h"
-#include "api/array_view.h"
-#include "api/crypto/frame_decryptor_interface.h"
-#include "api/sequence_checker.h"
-#include "api/transport/field_trial_based_config.h"
-#include "api/units/timestamp.h"
-#include "api/video/color_space.h"
-#include "api/video/video_codec_type.h"
-#include "call/rtp_packet_sink_interface.h"
-#include "call/syncable.h"
-#include "call/video_receive_stream.h"
-#include "modules/rtp_rtcp/include/receive_statistics.h"
-#include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h"
-#include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
-#include "modules/rtp_rtcp/include/rtp_rtcp.h"
-#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
-#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h"
-#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h"
-#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
-#include "modules/rtp_rtcp/source/rtp_packet_received.h"
-#include "modules/rtp_rtcp/source/rtp_video_header.h"
-#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h"
-#include "modules/video_coding/h264_sps_pps_tracker.h"
-#include "modules/video_coding/loss_notification_controller.h"
-#include "modules/video_coding/packet_buffer.h"
-#include "modules/video_coding/rtp_frame_reference_finder.h"
-#include "rtc_base/experiments/field_trial_parser.h"
-#include "rtc_base/numerics/sequence_number_util.h"
-#include "rtc_base/synchronization/mutex.h"
-#include "rtc_base/system/no_unique_address.h"
-#include "rtc_base/thread_annotations.h"
-#include "video/buffered_frame_decryptor.h"
-#include "video/rtp_video_stream_receiver_frame_transformer_delegate.h"
-#include "video/unique_timestamp_counter.h"
-
-namespace webrtc {
-
-class DEPRECATED_NackModule;
-class PacketRouter;
-class ProcessThread;
-class ReceiveStatistics;
-class ReceiveStatisticsProxy;
-class RtcpRttStats;
-class RtpPacketReceived;
-class Transport;
-class UlpfecReceiver;
-
-class RtpVideoStreamReceiver : public LossNotificationSender,
- public RecoveredPacketReceiver,
- public RtpPacketSinkInterface,
- public KeyFrameRequestSender,
- public OnDecryptedFrameCallback,
- public OnDecryptionStatusChangeCallback,
- public RtpVideoFrameReceiver {
- public:
- // A complete frame is a frame which has received all its packets and all its
- // references are known.
- class OnCompleteFrameCallback {
- public:
- virtual ~OnCompleteFrameCallback() {}
- virtual void OnCompleteFrame(std::unique_ptr<EncodedFrame> frame) = 0;
- };
-
- // DEPRECATED due to dependency on ReceiveStatisticsProxy.
- RtpVideoStreamReceiver(
- Clock* clock,
- Transport* transport,
- RtcpRttStats* rtt_stats,
- // The packet router is optional; if provided, the RtpRtcp module for this
- // stream is registered as a candidate for sending REMB and transport
- // feedback.
- PacketRouter* packet_router,
- const VideoReceiveStreamInterface::Config* config,
- ReceiveStatistics* rtp_receive_statistics,
- ReceiveStatisticsProxy* receive_stats_proxy,
- ProcessThread* process_thread,
- NackSender* nack_sender,
- // The KeyFrameRequestSender is optional; if not provided, key frame
- // requests are sent via the internal RtpRtcp module.
- KeyFrameRequestSender* keyframe_request_sender,
- OnCompleteFrameCallback* complete_frame_callback,
- rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
- rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
- const FieldTrialsView* field_trials = nullptr);
-
- RtpVideoStreamReceiver(
- Clock* clock,
- Transport* transport,
- RtcpRttStats* rtt_stats,
- // The packet router is optional; if provided, the RtpRtcp module for this
- // stream is registered as a candidate for sending REMB and transport
- // feedback.
- PacketRouter* packet_router,
- const VideoReceiveStreamInterface::Config* config,
- ReceiveStatistics* rtp_receive_statistics,
- RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
- RtcpCnameCallback* rtcp_cname_callback,
- ProcessThread* process_thread,
- NackSender* nack_sender,
- // The KeyFrameRequestSender is optional; if not provided, key frame
- // requests are sent via the internal RtpRtcp module.
- KeyFrameRequestSender* keyframe_request_sender,
- OnCompleteFrameCallback* complete_frame_callback,
- rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
- rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
- const FieldTrialsView* field_trials = nullptr);
- ~RtpVideoStreamReceiver() override;
-
- void AddReceiveCodec(uint8_t payload_type,
- VideoCodecType codec_type,
- const std::map<std::string, std::string>& codec_params,
- bool raw_payload);
-
- void StartReceive();
- void StopReceive();
-
- // Produces the transport-related timestamps; current_delay_ms is left unset.
- absl::optional<Syncable::Info> GetSyncInfo() const;
-
- bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length);
-
- void FrameContinuous(int64_t seq_num);
-
- void FrameDecoded(int64_t seq_num);
-
- void SignalNetworkState(NetworkState state);
-
- // Returns number of different frames seen.
- int GetUniqueFramesSeen() const {
- RTC_DCHECK_RUN_ON(&worker_task_checker_);
- return frame_counter_.GetUniqueSeen();
- }
-
- // Implements RtpPacketSinkInterface.
- void OnRtpPacket(const RtpPacketReceived& packet) override;
-
- // Public only for tests.
- void OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload,
- const RtpPacketReceived& rtp_packet,
- const RTPVideoHeader& video);
-
- // Implements RecoveredPacketReceiver.
- void OnRecoveredPacket(const uint8_t* packet, size_t packet_length) override;
-
- // Send an RTCP keyframe request.
- void RequestKeyFrame() override;
-
- // Implements LossNotificationSender.
- void SendLossNotification(uint16_t last_decoded_seq_num,
- uint16_t last_received_seq_num,
- bool decodability_flag,
- bool buffering_allowed) override;
-
- bool IsUlpfecEnabled() const;
- bool IsRetransmissionsEnabled() const;
-
- // Returns true if a decryptor is attached and frames can be decrypted.
- // Updated by OnDecryptionStatusChangeCallback. Note this refers to Frame
- // Decryption not SRTP.
- bool IsDecryptable() const;
-
- // Don't use, still experimental.
- void RequestPacketRetransmit(const std::vector<uint16_t>& sequence_numbers);
-
- void OnCompleteFrames(RtpFrameReferenceFinder::ReturnVector frames);
-
- // Implements OnDecryptedFrameCallback.
- void OnDecryptedFrame(std::unique_ptr<RtpFrameObject> frame) override;
-
- // Implements OnDecryptionStatusChangeCallback.
- void OnDecryptionStatusChange(
- FrameDecryptorInterface::Status status) override;
-
- // Optionally set a frame decryptor after a stream has started. This will not
- // reset the decoder state.
- void SetFrameDecryptor(
- rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor);
-
- // Sets a frame transformer after a stream has started, if no transformer
- // has previously been set. Does not reset the decoder state.
- void SetDepacketizerToDecoderFrameTransformer(
- rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
-
- // Called by VideoReceiveStreamInterface when stats are updated.
- void UpdateRtt(int64_t max_rtt_ms);
-
- absl::optional<int64_t> LastReceivedPacketMs() const;
- absl::optional<int64_t> LastReceivedKeyframePacketMs() const;
-
- // RtpDemuxer only forwards a given RTP packet to one sink. However, some
- // sinks, such as FlexFEC, might wish to be informed of all of the packets
- // a given sink receives (or any set of sinks). They may do so by registering
- // themselves as secondary sinks.
- void AddSecondarySink(RtpPacketSinkInterface* sink);
- void RemoveSecondarySink(const RtpPacketSinkInterface* sink);
-
- private:
- // Implements RtpVideoFrameReceiver.
- void ManageFrame(std::unique_ptr<RtpFrameObject> frame) override;
-
- // Used for buffering RTCP feedback messages and sending them all together.
- // Note:
- // 1. Key frame requests and NACKs are mutually exclusive, with the
- // former taking precedence over the latter.
- // 2. Loss notifications are orthogonal to either. (That is, may be sent
- // alongside either.)
- class RtcpFeedbackBuffer : public KeyFrameRequestSender,
- public NackSender,
- public LossNotificationSender {
- public:
- RtcpFeedbackBuffer(KeyFrameRequestSender* key_frame_request_sender,
- NackSender* nack_sender,
- LossNotificationSender* loss_notification_sender);
-
- ~RtcpFeedbackBuffer() override = default;
-
- // KeyFrameRequestSender implementation.
- void RequestKeyFrame() RTC_LOCKS_EXCLUDED(mutex_) override;
-
- // NackSender implementation.
- void SendNack(const std::vector<uint16_t>& sequence_numbers,
- bool buffering_allowed) RTC_LOCKS_EXCLUDED(mutex_) override;
-
- // LossNotificationSender implementation.
- void SendLossNotification(uint16_t last_decoded_seq_num,
- uint16_t last_received_seq_num,
- bool decodability_flag,
- bool buffering_allowed)
- RTC_LOCKS_EXCLUDED(mutex_) override;
-
- // Send all RTCP feedback messages buffered thus far.
- void SendBufferedRtcpFeedback() RTC_LOCKS_EXCLUDED(mutex_);
-
- private:
- // LNTF-related state.
- struct LossNotificationState {
- LossNotificationState(uint16_t last_decoded_seq_num,
- uint16_t last_received_seq_num,
- bool decodability_flag)
- : last_decoded_seq_num(last_decoded_seq_num),
- last_received_seq_num(last_received_seq_num),
- decodability_flag(decodability_flag) {}
-
- uint16_t last_decoded_seq_num;
- uint16_t last_received_seq_num;
- bool decodability_flag;
- };
- struct ConsumedRtcpFeedback {
- bool request_key_frame = false;
- std::vector<uint16_t> nack_sequence_numbers;
- absl::optional<LossNotificationState> lntf_state;
- };
-
- ConsumedRtcpFeedback ConsumeRtcpFeedback() RTC_LOCKS_EXCLUDED(mutex_);
- ConsumedRtcpFeedback ConsumeRtcpFeedbackLocked()
- RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
- // This method is called both with and without mutex_ held.
- void SendRtcpFeedback(ConsumedRtcpFeedback feedback);
-
- KeyFrameRequestSender* const key_frame_request_sender_;
- NackSender* const nack_sender_;
- LossNotificationSender* const loss_notification_sender_;
-
- // NACKs are accessible from two threads due to nack_module_ being a module.
- Mutex mutex_;
-
- // Key-frame-request-related state.
- bool request_key_frame_ RTC_GUARDED_BY(mutex_);
-
- // NACK-related state.
- std::vector<uint16_t> nack_sequence_numbers_ RTC_GUARDED_BY(mutex_);
-
- absl::optional<LossNotificationState> lntf_state_ RTC_GUARDED_BY(mutex_);
- };
- enum ParseGenericDependenciesResult {
- kDropPacket,
- kHasGenericDescriptor,
- kNoGenericDescriptor
- };
-
- // Entry point doing non-stats work for a received packet. Called
- // for the same packet both before and after RED decapsulation.
- void ReceivePacket(const RtpPacketReceived& packet);
- // Parses and handles RED headers.
- // This function assumes that it's being called from only one thread.
- void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet);
- void NotifyReceiverOfEmptyPacket(uint16_t seq_num);
- void UpdateHistograms();
- bool IsRedEnabled() const;
- void InsertSpsPpsIntoTracker(uint8_t payload_type);
- void OnInsertedPacket(video_coding::PacketBuffer::InsertResult result);
- ParseGenericDependenciesResult ParseGenericDependenciesExtension(
- const RtpPacketReceived& rtp_packet,
- RTPVideoHeader* video_header) RTC_RUN_ON(worker_task_checker_);
- void OnAssembledFrame(std::unique_ptr<RtpFrameObject> frame)
- RTC_LOCKS_EXCLUDED(packet_buffer_lock_);
- void UpdatePacketReceiveTimestamps(const RtpPacketReceived& packet,
- bool is_keyframe)
- RTC_RUN_ON(worker_task_checker_);
-
- const FieldTrialsView& field_trials_;
- FieldTrialBasedConfig owned_field_trials_;
-
- Clock* const clock_;
- // Ownership of this object lies with VideoReceiveStreamInterface, which owns
- // `this`.
- const VideoReceiveStreamInterface::Config& config_;
- PacketRouter* const packet_router_;
- ProcessThread* const process_thread_;
-
- RemoteNtpTimeEstimator ntp_estimator_;
-
- RtpHeaderExtensionMap rtp_header_extensions_;
- // Set by the field trial WebRTC-ForcePlayoutDelay to override any playout
- // delay that is specified in the received packets.
- FieldTrialOptional<int> forced_playout_delay_max_ms_;
- FieldTrialOptional<int> forced_playout_delay_min_ms_;
- ReceiveStatistics* const rtp_receive_statistics_;
- std::unique_ptr<UlpfecReceiver> ulpfec_receiver_;
-
- RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_task_checker_;
- bool receiving_ RTC_GUARDED_BY(worker_task_checker_);
- int64_t last_packet_log_ms_ RTC_GUARDED_BY(worker_task_checker_);
-
- const std::unique_ptr<RtpRtcp> rtp_rtcp_;
-
- OnCompleteFrameCallback* complete_frame_callback_;
- KeyFrameRequestSender* const keyframe_request_sender_;
-
- RtcpFeedbackBuffer rtcp_feedback_buffer_;
- std::unique_ptr<DEPRECATED_NackModule> nack_module_;
- std::unique_ptr<LossNotificationController> loss_notification_controller_;
-
- mutable Mutex packet_buffer_lock_;
- video_coding::PacketBuffer packet_buffer_ RTC_GUARDED_BY(packet_buffer_lock_);
- UniqueTimestampCounter frame_counter_ RTC_GUARDED_BY(worker_task_checker_);
- SeqNumUnwrapper<uint16_t> frame_id_unwrapper_
- RTC_GUARDED_BY(worker_task_checker_);
-
- // Video structure provided in the dependency descriptor in a first packet
- // of a key frame. It is required to parse dependency descriptor in the
- // following delta packets.
- std::unique_ptr<FrameDependencyStructure> video_structure_
- RTC_GUARDED_BY(worker_task_checker_);
- // Frame id of the last frame with the attached video structure.
- // absl::nullopt when `video_structure_ == nullptr`;
- absl::optional<int64_t> video_structure_frame_id_
- RTC_GUARDED_BY(worker_task_checker_);
-
- Mutex reference_finder_lock_;
- std::unique_ptr<RtpFrameReferenceFinder> reference_finder_
- RTC_GUARDED_BY(reference_finder_lock_);
- absl::optional<VideoCodecType> current_codec_;
- uint32_t last_assembled_frame_rtp_timestamp_;
-
- Mutex last_seq_num_mutex_;
- std::map<int64_t, uint16_t> last_seq_num_for_pic_id_
- RTC_GUARDED_BY(last_seq_num_mutex_);
- video_coding::H264SpsPpsTracker tracker_;
-
- // Maps payload id to the depacketizer.
- std::map<uint8_t, std::unique_ptr<VideoRtpDepacketizer>> payload_type_map_;
-
- // TODO(johan): Remove pt_codec_params_ once
- // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved.
- // Maps a payload type to a map of out-of-band supplied codec parameters.
- std::map<uint8_t, std::map<std::string, std::string>> pt_codec_params_;
- int16_t last_payload_type_ = -1;
-
- bool has_received_frame_;
-
- std::vector<RtpPacketSinkInterface*> secondary_sinks_
- RTC_GUARDED_BY(worker_task_checker_);
-
- // Info for GetSyncInfo is updated on network or worker thread, and queried on
- // the worker thread.
- mutable Mutex sync_info_lock_;
- absl::optional<uint32_t> last_received_rtp_timestamp_
- RTC_GUARDED_BY(sync_info_lock_);
- absl::optional<uint32_t> last_received_keyframe_rtp_timestamp_
- RTC_GUARDED_BY(sync_info_lock_);
- absl::optional<Timestamp> last_received_rtp_system_time_
- RTC_GUARDED_BY(sync_info_lock_);
- absl::optional<Timestamp> last_received_keyframe_rtp_system_time_
- RTC_GUARDED_BY(sync_info_lock_);
-
- // Used to validate the buffered frame decryptor is always run on the correct
- // thread.
- SequenceChecker network_tc_;
- // Handles incoming encrypted frames and forwards them to the
- // rtp_reference_finder if they are decryptable.
- std::unique_ptr<BufferedFrameDecryptor> buffered_frame_decryptor_
- RTC_PT_GUARDED_BY(network_tc_);
- std::atomic<bool> frames_decryptable_;
- absl::optional<ColorSpace> last_color_space_;
-
- AbsoluteCaptureTimeInterpolator absolute_capture_time_interpolator_
- RTC_GUARDED_BY(worker_task_checker_);
-
- CaptureClockOffsetUpdater capture_clock_offset_updater_
- RTC_GUARDED_BY(worker_task_checker_);
-
- int64_t last_completed_picture_id_ = 0;
-
- rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate>
- frame_transformer_delegate_;
-
- SeqNumUnwrapper<uint16_t> rtp_seq_num_unwrapper_
- RTC_GUARDED_BY(packet_buffer_lock_);
- std::map<int64_t, RtpPacketInfo> packet_infos_
- RTC_GUARDED_BY(packet_buffer_lock_);
-};
-
-} // namespace webrtc
-
-#endif // VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_
diff --git a/video/rtp_video_stream_receiver_unittest.cc b/video/rtp_video_stream_receiver_unittest.cc
deleted file mode 100644
index bfe9a4c..0000000
--- a/video/rtp_video_stream_receiver_unittest.cc
+++ /dev/null
@@ -1,1238 +0,0 @@
-/*
- * Copyright 2017 The WebRTC Project Authors. All rights reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "video/rtp_video_stream_receiver.h"
-
-#include <memory>
-#include <utility>
-
-#include "api/video/video_codec_type.h"
-#include "api/video/video_frame_type.h"
-#include "call/test/mock_rtp_packet_sink_interface.h"
-#include "common_video/h264/h264_common.h"
-#include "media/base/media_constants.h"
-#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
-#include "modules/rtp_rtcp/source/rtp_format.h"
-#include "modules/rtp_rtcp/source/rtp_format_vp9.h"
-#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h"
-#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
-#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
-#include "modules/rtp_rtcp/source/rtp_packet_received.h"
-#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
-#include "modules/utility/include/process_thread.h"
-#include "modules/video_coding/frame_object.h"
-#include "modules/video_coding/include/video_coding_defines.h"
-#include "modules/video_coding/rtp_frame_reference_finder.h"
-#include "rtc_base/byte_buffer.h"
-#include "rtc_base/logging.h"
-#include "system_wrappers/include/clock.h"
-#include "test/gmock.h"
-#include "test/gtest.h"
-#include "test/mock_frame_transformer.h"
-#include "test/mock_transport.h"
-#include "test/scoped_key_value_config.h"
-
-using ::testing::_;
-using ::testing::ElementsAre;
-using ::testing::Invoke;
-using ::testing::SizeIs;
-using ::testing::Values;
-
-namespace webrtc {
-
-namespace {
-
-const uint8_t kH264StartCode[] = {0x00, 0x00, 0x00, 0x01};
-
-std::vector<uint64_t> GetAbsoluteCaptureTimestamps(const EncodedFrame* frame) {
- std::vector<uint64_t> result;
- for (const auto& packet_info : frame->PacketInfos()) {
- if (packet_info.absolute_capture_time()) {
- result.push_back(
- packet_info.absolute_capture_time()->absolute_capture_timestamp);
- }
- }
- return result;
-}
-
-RTPVideoHeader GetGenericVideoHeader(VideoFrameType frame_type) {
- RTPVideoHeader video_header;
- video_header.is_first_packet_in_frame = true;
- video_header.is_last_packet_in_frame = true;
- video_header.codec = kVideoCodecGeneric;
- video_header.frame_type = frame_type;
- return video_header;
-}
-
-class MockNackSender : public NackSender {
- public:
- MOCK_METHOD(void,
- SendNack,
- (const std::vector<uint16_t>& sequence_numbers,
- bool buffering_allowed),
- (override));
-};
-
-class MockKeyFrameRequestSender : public KeyFrameRequestSender {
- public:
- MOCK_METHOD(void, RequestKeyFrame, (), (override));
-};
-
-class MockOnCompleteFrameCallback
- : public RtpVideoStreamReceiver::OnCompleteFrameCallback {
- public:
- MOCK_METHOD(void, DoOnCompleteFrame, (EncodedFrame*), ());
- MOCK_METHOD(void, DoOnCompleteFrameFailNullptr, (EncodedFrame*), ());
- MOCK_METHOD(void, DoOnCompleteFrameFailLength, (EncodedFrame*), ());
- MOCK_METHOD(void, DoOnCompleteFrameFailBitstream, (EncodedFrame*), ());
- void OnCompleteFrame(std::unique_ptr<EncodedFrame> frame) override {
- if (!frame) {
- DoOnCompleteFrameFailNullptr(nullptr);
- return;
- }
- EXPECT_EQ(buffer_.Length(), frame->size());
- if (buffer_.Length() != frame->size()) {
- DoOnCompleteFrameFailLength(frame.get());
- return;
- }
- if (frame->size() != buffer_.Length() ||
- memcmp(buffer_.Data(), frame->data(), buffer_.Length()) != 0) {
- DoOnCompleteFrameFailBitstream(frame.get());
- return;
- }
- DoOnCompleteFrame(frame.get());
- }
-
- void ClearExpectedBitstream() { buffer_.Clear(); }
-
- void AppendExpectedBitstream(const uint8_t data[], size_t size_in_bytes) {
- // TODO(Johan): Let rtc::ByteBuffer handle uint8_t* instead of char*.
- buffer_.WriteBytes(reinterpret_cast<const char*>(data), size_in_bytes);
- }
- rtc::ByteBufferWriter buffer_;
-};
-
-constexpr uint32_t kSsrc = 111;
-constexpr uint16_t kSequenceNumber = 222;
-constexpr int kPayloadType = 100;
-constexpr int kRedPayloadType = 125;
-
-std::unique_ptr<RtpPacketReceived> CreateRtpPacketReceived() {
- auto packet = std::make_unique<RtpPacketReceived>();
- packet->SetSsrc(kSsrc);
- packet->SetSequenceNumber(kSequenceNumber);
- packet->SetPayloadType(kPayloadType);
- return packet;
-}
-
-MATCHER_P(SamePacketAs, other, "") {
- return arg.Ssrc() == other.Ssrc() &&
- arg.SequenceNumber() == other.SequenceNumber();
-}
-
-} // namespace
-
-class RtpVideoStreamReceiverTest : public ::testing::Test {
- public:
- RtpVideoStreamReceiverTest() : RtpVideoStreamReceiverTest("") {}
- explicit RtpVideoStreamReceiverTest(std::string field_trials)
- : field_trials_(field_trials),
- config_(CreateConfig()),
- process_thread_(ProcessThread::Create("TestThread")) {
- rtp_receive_statistics_ =
- ReceiveStatistics::Create(Clock::GetRealTimeClock());
- rtp_video_stream_receiver_ = std::make_unique<RtpVideoStreamReceiver>(
- Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_,
- rtp_receive_statistics_.get(), nullptr, nullptr, process_thread_.get(),
- &mock_nack_sender_, &mock_key_frame_request_sender_,
- &mock_on_complete_frame_callback_, nullptr, nullptr, &field_trials_);
- rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType,
- kVideoCodecGeneric, {},
- /*raw_payload=*/false);
- }
-
- RTPVideoHeader GetDefaultH264VideoHeader() {
- RTPVideoHeader video_header;
- video_header.codec = kVideoCodecH264;
- video_header.video_type_header.emplace<RTPVideoHeaderH264>();
- return video_header;
- }
-
- // TODO(Johan): refactor h264_sps_pps_tracker_unittests.cc to avoid duplicate
- // code.
- void AddSps(RTPVideoHeader* video_header,
- uint8_t sps_id,
- rtc::CopyOnWriteBuffer* data) {
- NaluInfo info;
- info.type = H264::NaluType::kSps;
- info.sps_id = sps_id;
- info.pps_id = -1;
- data->AppendData<uint8_t, 2>({H264::NaluType::kSps, sps_id});
- auto& h264 = absl::get<RTPVideoHeaderH264>(video_header->video_type_header);
- h264.nalus[h264.nalus_length++] = info;
- }
-
- void AddPps(RTPVideoHeader* video_header,
- uint8_t sps_id,
- uint8_t pps_id,
- rtc::CopyOnWriteBuffer* data) {
- NaluInfo info;
- info.type = H264::NaluType::kPps;
- info.sps_id = sps_id;
- info.pps_id = pps_id;
- data->AppendData<uint8_t, 2>({H264::NaluType::kPps, pps_id});
- auto& h264 = absl::get<RTPVideoHeaderH264>(video_header->video_type_header);
- h264.nalus[h264.nalus_length++] = info;
- }
-
- void AddIdr(RTPVideoHeader* video_header, int pps_id) {
- NaluInfo info;
- info.type = H264::NaluType::kIdr;
- info.sps_id = -1;
- info.pps_id = pps_id;
- auto& h264 = absl::get<RTPVideoHeaderH264>(video_header->video_type_header);
- h264.nalus[h264.nalus_length++] = info;
- }
-
- protected:
- static VideoReceiveStreamInterface::Config CreateConfig() {
- VideoReceiveStreamInterface::Config config(nullptr);
- config.rtp.remote_ssrc = 1111;
- config.rtp.local_ssrc = 2222;
- config.rtp.red_payload_type = kRedPayloadType;
- return config;
- }
-
- webrtc::test::ScopedKeyValueConfig field_trials_;
- VideoReceiveStreamInterface::Config config_;
- MockNackSender mock_nack_sender_;
- MockKeyFrameRequestSender mock_key_frame_request_sender_;
- MockTransport mock_transport_;
- MockOnCompleteFrameCallback mock_on_complete_frame_callback_;
- std::unique_ptr<ProcessThread> process_thread_;
- std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_;
- std::unique_ptr<RtpVideoStreamReceiver> rtp_video_stream_receiver_;
-};
-
-TEST_F(RtpVideoStreamReceiverTest, CacheColorSpaceFromLastPacketOfKeyframe) {
- // Test that color space is cached from the last packet of a key frame and
- // that it's not reset by padding packets without color space.
- constexpr int kVp9PayloadType = 99;
- const ColorSpace kColorSpace(
- ColorSpace::PrimaryID::kFILM, ColorSpace::TransferID::kBT2020_12,
- ColorSpace::MatrixID::kBT2020_NCL, ColorSpace::RangeID::kFull);
- const std::vector<uint8_t> kKeyFramePayload = {0, 1, 2, 3, 4, 5,
- 6, 7, 8, 9, 10};
- const std::vector<uint8_t> kDeltaFramePayload = {0, 1, 2, 3, 4};
-
- // Anonymous helper class that generates received packets.
- class {
- public:
- void SetPayload(const std::vector<uint8_t>& payload,
- VideoFrameType video_frame_type) {
- video_frame_type_ = video_frame_type;
- RtpPacketizer::PayloadSizeLimits pay_load_size_limits;
- // Reduce max payload length to make sure the key frame generates two
- // packets.
- pay_load_size_limits.max_payload_len = 8;
- RTPVideoHeaderVP9 rtp_video_header_vp9;
- rtp_video_header_vp9.InitRTPVideoHeaderVP9();
- rtp_video_header_vp9.inter_pic_predicted =
- (video_frame_type == VideoFrameType::kVideoFrameDelta);
- rtp_packetizer_ = std::make_unique<RtpPacketizerVp9>(
- payload, pay_load_size_limits, rtp_video_header_vp9);
- }
-
- size_t NumPackets() { return rtp_packetizer_->NumPackets(); }
- void SetColorSpace(const ColorSpace& color_space) {
- color_space_ = color_space;
- }
-
- RtpPacketReceived NextPacket() {
- RtpHeaderExtensionMap extension_map;
- extension_map.Register<ColorSpaceExtension>(1);
- RtpPacketToSend packet_to_send(&extension_map);
- packet_to_send.SetSequenceNumber(sequence_number_++);
- packet_to_send.SetSsrc(kSsrc);
- packet_to_send.SetPayloadType(kVp9PayloadType);
- bool include_color_space =
- (rtp_packetizer_->NumPackets() == 1u &&
- video_frame_type_ == VideoFrameType::kVideoFrameKey);
- if (include_color_space) {
- EXPECT_TRUE(
- packet_to_send.SetExtension<ColorSpaceExtension>(color_space_));
- }
- rtp_packetizer_->NextPacket(&packet_to_send);
-
- RtpPacketReceived received_packet(&extension_map);
- received_packet.Parse(packet_to_send.data(), packet_to_send.size());
- return received_packet;
- }
-
- private:
- uint16_t sequence_number_ = 0;
- VideoFrameType video_frame_type_;
- ColorSpace color_space_;
- std::unique_ptr<RtpPacketizer> rtp_packetizer_;
- } received_packet_generator;
- received_packet_generator.SetColorSpace(kColorSpace);
-
- // Prepare the receiver for VP9.
- std::map<std::string, std::string> codec_params;
- rtp_video_stream_receiver_->AddReceiveCodec(kVp9PayloadType, kVideoCodecVP9,
- codec_params,
- /*raw_payload=*/false);
-
- // Generate key frame packets.
- received_packet_generator.SetPayload(kKeyFramePayload,
- VideoFrameType::kVideoFrameKey);
- EXPECT_EQ(received_packet_generator.NumPackets(), 2u);
- RtpPacketReceived key_frame_packet1 = received_packet_generator.NextPacket();
- RtpPacketReceived key_frame_packet2 = received_packet_generator.NextPacket();
-
- // Generate delta frame packet.
- received_packet_generator.SetPayload(kDeltaFramePayload,
- VideoFrameType::kVideoFrameDelta);
- EXPECT_EQ(received_packet_generator.NumPackets(), 1u);
- RtpPacketReceived delta_frame_packet = received_packet_generator.NextPacket();
-
- rtp_video_stream_receiver_->StartReceive();
- mock_on_complete_frame_callback_.AppendExpectedBitstream(
- kKeyFramePayload.data(), kKeyFramePayload.size());
-
- // Send the key frame and expect a callback with color space information.
- EXPECT_FALSE(key_frame_packet1.GetExtension<ColorSpaceExtension>());
- EXPECT_TRUE(key_frame_packet2.GetExtension<ColorSpaceExtension>());
- rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet1);
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
- .WillOnce(Invoke([kColorSpace](EncodedFrame* frame) {
- ASSERT_TRUE(frame->EncodedImage().ColorSpace());
- EXPECT_EQ(*frame->EncodedImage().ColorSpace(), kColorSpace);
- }));
- rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet2);
- // Resend the first key frame packet to simulate padding for example.
- rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet1);
-
- mock_on_complete_frame_callback_.ClearExpectedBitstream();
- mock_on_complete_frame_callback_.AppendExpectedBitstream(
- kDeltaFramePayload.data(), kDeltaFramePayload.size());
-
- // Expect delta frame to have color space set even though color space not
- // included in the RTP packet.
- EXPECT_FALSE(delta_frame_packet.GetExtension<ColorSpaceExtension>());
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
- .WillOnce(Invoke([kColorSpace](EncodedFrame* frame) {
- ASSERT_TRUE(frame->EncodedImage().ColorSpace());
- EXPECT_EQ(*frame->EncodedImage().ColorSpace(), kColorSpace);
- }));
- rtp_video_stream_receiver_->OnRtpPacket(delta_frame_packet);
-}
-
-TEST_F(RtpVideoStreamReceiverTest, GenericKeyFrame) {
- RtpPacketReceived rtp_packet;
- rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'});
- rtp_packet.SetPayloadType(kPayloadType);
- rtp_packet.SetSequenceNumber(1);
- RTPVideoHeader video_header =
- GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
- mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
- data.size());
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
- rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
- video_header);
-}
-
-TEST_F(RtpVideoStreamReceiverTest, PacketInfoIsPropagatedIntoVideoFrames) {
- constexpr uint64_t kAbsoluteCaptureTimestamp = 12;
- constexpr int kId0 = 1;
-
- RtpHeaderExtensionMap extension_map;
- extension_map.Register<AbsoluteCaptureTimeExtension>(kId0);
- RtpPacketReceived rtp_packet(&extension_map);
- rtp_packet.SetPayloadType(kPayloadType);
- rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'});
- rtp_packet.SetSequenceNumber(1);
- rtp_packet.SetTimestamp(1);
- rtp_packet.SetSsrc(kSsrc);
- rtp_packet.SetExtension<AbsoluteCaptureTimeExtension>(
- AbsoluteCaptureTime{kAbsoluteCaptureTimestamp,
- /*estimated_capture_clock_offset=*/absl::nullopt});
-
- RTPVideoHeader video_header =
- GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
- mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
- data.size());
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
- .WillOnce(Invoke([kAbsoluteCaptureTimestamp](EncodedFrame* frame) {
- EXPECT_THAT(GetAbsoluteCaptureTimestamps(frame),
- ElementsAre(kAbsoluteCaptureTimestamp));
- }));
- rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
- video_header);
-}
-
-TEST_F(RtpVideoStreamReceiverTest,
- MissingAbsoluteCaptureTimeIsFilledWithExtrapolatedValue) {
- constexpr uint64_t kAbsoluteCaptureTimestamp = 12;
- constexpr int kId0 = 1;
-
- RtpHeaderExtensionMap extension_map;
- extension_map.Register<AbsoluteCaptureTimeExtension>(kId0);
- RtpPacketReceived rtp_packet(&extension_map);
- rtp_packet.SetPayloadType(kPayloadType);
-
- rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'});
- uint16_t sequence_number = 1;
- uint32_t rtp_timestamp = 1;
- rtp_packet.SetSequenceNumber(sequence_number);
- rtp_packet.SetTimestamp(rtp_timestamp);
- rtp_packet.SetSsrc(kSsrc);
- rtp_packet.SetExtension<AbsoluteCaptureTimeExtension>(
- AbsoluteCaptureTime{kAbsoluteCaptureTimestamp,
- /*estimated_capture_clock_offset=*/absl::nullopt});
-
- RTPVideoHeader video_header =
- GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
- mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
- data.size());
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
- rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
- video_header);
-
- // Rtp packet without absolute capture time.
- rtp_packet = RtpPacketReceived(&extension_map);
- rtp_packet.SetPayloadType(kPayloadType);
- rtp_packet.SetSequenceNumber(++sequence_number);
- rtp_packet.SetTimestamp(++rtp_timestamp);
- rtp_packet.SetSsrc(kSsrc);
-
- // There is no absolute capture time in the second packet.
- // Expect rtp video stream receiver to extrapolate it for the resulting video
- // frame using absolute capture time from the previous packet.
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
- .WillOnce(Invoke([](EncodedFrame* frame) {
- EXPECT_THAT(GetAbsoluteCaptureTimestamps(frame), SizeIs(1));
- }));
- rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
- video_header);
-}
-
-TEST_F(RtpVideoStreamReceiverTest, NoInfiniteRecursionOnEncapsulatedRedPacket) {
- const std::vector<uint8_t> data({
- 0x80, // RTP version.
- kRedPayloadType, // Payload type.
- 0, 0, 0, 0, 0, 0, // Don't care.
- 0, 0, 0x4, 0x57, // SSRC
- kRedPayloadType, // RED header.
- 0, 0, 0, 0, 0 // Don't care.
- });
- RtpPacketReceived packet;
- EXPECT_TRUE(packet.Parse(data.data(), data.size()));
- rtp_video_stream_receiver_->StartReceive();
- rtp_video_stream_receiver_->OnRtpPacket(packet);
-}
-
-TEST_F(RtpVideoStreamReceiverTest,
- DropsPacketWithRedPayloadTypeAndEmptyPayload) {
- const uint8_t kRedPayloadType = 125;
- config_.rtp.red_payload_type = kRedPayloadType;
- SetUp(); // re-create rtp_video_stream_receiver with red payload type.
- // clang-format off
- const uint8_t data[] = {
- 0x80, // RTP version.
- kRedPayloadType, // Payload type.
- 0, 0, 0, 0, 0, 0, // Don't care.
- 0, 0, 0x4, 0x57, // SSRC
- // Empty rtp payload.
- };
- // clang-format on
- RtpPacketReceived packet;
- // Manually convert to CopyOnWriteBuffer to be sure capacity == size
- // and asan bot can catch read buffer overflow.
- EXPECT_TRUE(packet.Parse(rtc::CopyOnWriteBuffer(data)));
- rtp_video_stream_receiver_->StartReceive();
- rtp_video_stream_receiver_->OnRtpPacket(packet);
- // Expect asan doesn't find anything.
-}
-
-TEST_F(RtpVideoStreamReceiverTest, GenericKeyFrameBitstreamError) {
- RtpPacketReceived rtp_packet;
- rtp_packet.SetPayloadType(kPayloadType);
- rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'});
- rtp_packet.SetSequenceNumber(1);
- RTPVideoHeader video_header =
- GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
- constexpr uint8_t expected_bitsteam[] = {1, 2, 3, 0xff};
- mock_on_complete_frame_callback_.AppendExpectedBitstream(
- expected_bitsteam, sizeof(expected_bitsteam));
- EXPECT_CALL(mock_on_complete_frame_callback_,
- DoOnCompleteFrameFailBitstream(_));
- rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
- video_header);
-}
-
-class RtpVideoStreamReceiverTestH264
- : public RtpVideoStreamReceiverTest,
- public ::testing::WithParamInterface<std::string> {
- protected:
- RtpVideoStreamReceiverTestH264() : RtpVideoStreamReceiverTest(GetParam()) {}
-};
-
-INSTANTIATE_TEST_SUITE_P(SpsPpsIdrIsKeyframe,
- RtpVideoStreamReceiverTestH264,
- Values("", "WebRTC-SpsPpsIdrIsH264Keyframe/Enabled/"));
-
-TEST_P(RtpVideoStreamReceiverTestH264, InBandSpsPps) {
- rtc::CopyOnWriteBuffer sps_data;
- RtpPacketReceived rtp_packet;
- RTPVideoHeader sps_video_header = GetDefaultH264VideoHeader();
- AddSps(&sps_video_header, 0, &sps_data);
- rtp_packet.SetSequenceNumber(0);
- rtp_packet.SetPayloadType(kPayloadType);
- sps_video_header.is_first_packet_in_frame = true;
- sps_video_header.frame_type = VideoFrameType::kEmptyFrame;
- mock_on_complete_frame_callback_.AppendExpectedBitstream(
- kH264StartCode, sizeof(kH264StartCode));
- mock_on_complete_frame_callback_.AppendExpectedBitstream(sps_data.data(),
- sps_data.size());
- rtp_video_stream_receiver_->OnReceivedPayloadData(sps_data, rtp_packet,
- sps_video_header);
-
- rtc::CopyOnWriteBuffer pps_data;
- RTPVideoHeader pps_video_header = GetDefaultH264VideoHeader();
- AddPps(&pps_video_header, 0, 1, &pps_data);
- rtp_packet.SetSequenceNumber(1);
- pps_video_header.is_first_packet_in_frame = true;
- pps_video_header.frame_type = VideoFrameType::kEmptyFrame;
- mock_on_complete_frame_callback_.AppendExpectedBitstream(
- kH264StartCode, sizeof(kH264StartCode));
- mock_on_complete_frame_callback_.AppendExpectedBitstream(pps_data.data(),
- pps_data.size());
- rtp_video_stream_receiver_->OnReceivedPayloadData(pps_data, rtp_packet,
- pps_video_header);
-
- rtc::CopyOnWriteBuffer idr_data;
- RTPVideoHeader idr_video_header = GetDefaultH264VideoHeader();
- AddIdr(&idr_video_header, 1);
- rtp_packet.SetSequenceNumber(2);
- idr_video_header.is_first_packet_in_frame = true;
- idr_video_header.is_last_packet_in_frame = true;
- idr_video_header.frame_type = VideoFrameType::kVideoFrameKey;
- const uint8_t idr[] = {0x65, 1, 2, 3};
- idr_data.AppendData(idr);
- mock_on_complete_frame_callback_.AppendExpectedBitstream(
- kH264StartCode, sizeof(kH264StartCode));
- mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(),
- idr_data.size());
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
- rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet,
- idr_video_header);
-}
-
-TEST_P(RtpVideoStreamReceiverTestH264, OutOfBandFmtpSpsPps) {
- constexpr int kPayloadType = 99;
- std::map<std::string, std::string> codec_params;
- // Example parameter sets from https://tools.ietf.org/html/rfc3984#section-8.2
- // .
- codec_params.insert(
- {cricket::kH264FmtpSpropParameterSets, "Z0IACpZTBYmI,aMljiA=="});
- rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, kVideoCodecH264,
- codec_params,
- /*raw_payload=*/false);
- const uint8_t binary_sps[] = {0x67, 0x42, 0x00, 0x0a, 0x96,
- 0x53, 0x05, 0x89, 0x88};
- mock_on_complete_frame_callback_.AppendExpectedBitstream(
- kH264StartCode, sizeof(kH264StartCode));
- mock_on_complete_frame_callback_.AppendExpectedBitstream(binary_sps,
- sizeof(binary_sps));
- const uint8_t binary_pps[] = {0x68, 0xc9, 0x63, 0x88};
- mock_on_complete_frame_callback_.AppendExpectedBitstream(
- kH264StartCode, sizeof(kH264StartCode));
- mock_on_complete_frame_callback_.AppendExpectedBitstream(binary_pps,
- sizeof(binary_pps));
-
- RtpPacketReceived rtp_packet;
- RTPVideoHeader video_header = GetDefaultH264VideoHeader();
- AddIdr(&video_header, 0);
- rtp_packet.SetPayloadType(kPayloadType);
- rtp_packet.SetSequenceNumber(2);
- video_header.is_first_packet_in_frame = true;
- video_header.is_last_packet_in_frame = true;
- video_header.codec = kVideoCodecH264;
- video_header.frame_type = VideoFrameType::kVideoFrameKey;
- rtc::CopyOnWriteBuffer data({'1', '2', '3'});
- mock_on_complete_frame_callback_.AppendExpectedBitstream(
- kH264StartCode, sizeof(kH264StartCode));
- mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
- data.size());
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
- rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
- video_header);
-}
-
-TEST_P(RtpVideoStreamReceiverTestH264, ForceSpsPpsIdrIsKeyframe) {
- constexpr int kPayloadType = 99;
- std::map<std::string, std::string> codec_params;
- if (GetParam() ==
- "") { // Forcing can be done either with field trial or codec_params.
- codec_params.insert({cricket::kH264FmtpSpsPpsIdrInKeyframe, ""});
- }
- rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, kVideoCodecH264,
- codec_params,
- /*raw_payload=*/false);
- rtc::CopyOnWriteBuffer sps_data;
- RtpPacketReceived rtp_packet;
- RTPVideoHeader sps_video_header = GetDefaultH264VideoHeader();
- AddSps(&sps_video_header, 0, &sps_data);
- rtp_packet.SetSequenceNumber(0);
- rtp_packet.SetPayloadType(kPayloadType);
- sps_video_header.is_first_packet_in_frame = true;
- sps_video_header.frame_type = VideoFrameType::kEmptyFrame;
- mock_on_complete_frame_callback_.AppendExpectedBitstream(
- kH264StartCode, sizeof(kH264StartCode));
- mock_on_complete_frame_callback_.AppendExpectedBitstream(sps_data.data(),
- sps_data.size());
- rtp_video_stream_receiver_->OnReceivedPayloadData(sps_data, rtp_packet,
- sps_video_header);
-
- rtc::CopyOnWriteBuffer pps_data;
- RTPVideoHeader pps_video_header = GetDefaultH264VideoHeader();
- AddPps(&pps_video_header, 0, 1, &pps_data);
- rtp_packet.SetSequenceNumber(1);
- pps_video_header.is_first_packet_in_frame = true;
- pps_video_header.frame_type = VideoFrameType::kEmptyFrame;
- mock_on_complete_frame_callback_.AppendExpectedBitstream(
- kH264StartCode, sizeof(kH264StartCode));
- mock_on_complete_frame_callback_.AppendExpectedBitstream(pps_data.data(),
- pps_data.size());
- rtp_video_stream_receiver_->OnReceivedPayloadData(pps_data, rtp_packet,
- pps_video_header);
-
- rtc::CopyOnWriteBuffer idr_data;
- RTPVideoHeader idr_video_header = GetDefaultH264VideoHeader();
- AddIdr(&idr_video_header, 1);
- rtp_packet.SetSequenceNumber(2);
- idr_video_header.is_first_packet_in_frame = true;
- idr_video_header.is_last_packet_in_frame = true;
- idr_video_header.frame_type = VideoFrameType::kVideoFrameKey;
- const uint8_t idr[] = {0x65, 1, 2, 3};
- idr_data.AppendData(idr);
- mock_on_complete_frame_callback_.AppendExpectedBitstream(
- kH264StartCode, sizeof(kH264StartCode));
- mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(),
- idr_data.size());
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
- .WillOnce(
- [&](EncodedFrame* frame) { EXPECT_TRUE(frame->is_keyframe()); });
- rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet,
- idr_video_header);
- mock_on_complete_frame_callback_.ClearExpectedBitstream();
- mock_on_complete_frame_callback_.AppendExpectedBitstream(
- kH264StartCode, sizeof(kH264StartCode));
- mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(),
- idr_data.size());
- rtp_packet.SetSequenceNumber(3);
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
- .WillOnce(
- [&](EncodedFrame* frame) { EXPECT_FALSE(frame->is_keyframe()); });
- rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet,
- idr_video_header);
-}
-
-TEST_F(RtpVideoStreamReceiverTest, PaddingInMediaStream) {
- RtpPacketReceived rtp_packet;
- RTPVideoHeader video_header = GetDefaultH264VideoHeader();
- rtc::CopyOnWriteBuffer data({'1', '2', '3'});
- rtp_packet.SetPayloadType(kPayloadType);
- rtp_packet.SetSequenceNumber(2);
- video_header.is_first_packet_in_frame = true;
- video_header.is_last_packet_in_frame = true;
- video_header.codec = kVideoCodecGeneric;
- video_header.frame_type = VideoFrameType::kVideoFrameKey;
- mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
- data.size());
-
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
- rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
- video_header);
-
- rtp_packet.SetSequenceNumber(3);
- rtp_video_stream_receiver_->OnReceivedPayloadData({}, rtp_packet,
- video_header);
-
- rtp_packet.SetSequenceNumber(4);
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
- video_header.frame_type = VideoFrameType::kVideoFrameDelta;
- rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
- video_header);
-
- rtp_packet.SetSequenceNumber(6);
- rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
- video_header);
-
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
- rtp_packet.SetSequenceNumber(5);
- rtp_video_stream_receiver_->OnReceivedPayloadData({}, rtp_packet,
- video_header);
-}
-
-TEST_F(RtpVideoStreamReceiverTest, RequestKeyframeIfFirstFrameIsDelta) {
- RtpPacketReceived rtp_packet;
- rtp_packet.SetPayloadType(kPayloadType);
- rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'});
- rtp_packet.SetSequenceNumber(1);
- RTPVideoHeader video_header =
- GetGenericVideoHeader(VideoFrameType::kVideoFrameDelta);
- EXPECT_CALL(mock_key_frame_request_sender_, RequestKeyFrame());
- rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
- video_header);
-}
-
-TEST_F(RtpVideoStreamReceiverTest, RequestKeyframeWhenPacketBufferGetsFull) {
- constexpr int kPacketBufferMaxSize = 2048;
-
- RtpPacketReceived rtp_packet;
- rtp_packet.SetPayloadType(kPayloadType);
- rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'});
- RTPVideoHeader video_header =
- GetGenericVideoHeader(VideoFrameType::kVideoFrameDelta);
- // Incomplete frames so that the packet buffer is filling up.
- video_header.is_last_packet_in_frame = false;
- uint16_t start_sequence_number = 1234;
- rtp_packet.SetSequenceNumber(start_sequence_number);
- while (rtp_packet.SequenceNumber() - start_sequence_number <
- kPacketBufferMaxSize) {
- rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
- video_header);
- rtp_packet.SetSequenceNumber(rtp_packet.SequenceNumber() + 2);
- }
-
- EXPECT_CALL(mock_key_frame_request_sender_, RequestKeyFrame());
- rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
- video_header);
-}
-
-TEST_F(RtpVideoStreamReceiverTest, SecondarySinksGetRtpNotifications) {
- rtp_video_stream_receiver_->StartReceive();
-
- MockRtpPacketSink secondary_sink_1;
- MockRtpPacketSink secondary_sink_2;
-
- rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink_1);
- rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink_2);
-
- auto rtp_packet = CreateRtpPacketReceived();
- EXPECT_CALL(secondary_sink_1, OnRtpPacket(SamePacketAs(*rtp_packet)));
- EXPECT_CALL(secondary_sink_2, OnRtpPacket(SamePacketAs(*rtp_packet)));
-
- rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet);
-
- // Test tear-down.
- rtp_video_stream_receiver_->StopReceive();
- rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink_1);
- rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink_2);
-}
-
-TEST_F(RtpVideoStreamReceiverTest, RemovedSecondarySinksGetNoRtpNotifications) {
- rtp_video_stream_receiver_->StartReceive();
-
- MockRtpPacketSink secondary_sink;
-
- rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink);
- rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink);
-
- auto rtp_packet = CreateRtpPacketReceived();
-
- EXPECT_CALL(secondary_sink, OnRtpPacket(_)).Times(0);
-
- rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet);
-
- // Test tear-down.
- rtp_video_stream_receiver_->StopReceive();
-}
-
-TEST_F(RtpVideoStreamReceiverTest,
- OnlyRemovedSecondarySinksExcludedFromNotifications) {
- rtp_video_stream_receiver_->StartReceive();
-
- MockRtpPacketSink kept_secondary_sink;
- MockRtpPacketSink removed_secondary_sink;
-
- rtp_video_stream_receiver_->AddSecondarySink(&kept_secondary_sink);
- rtp_video_stream_receiver_->AddSecondarySink(&removed_secondary_sink);
- rtp_video_stream_receiver_->RemoveSecondarySink(&removed_secondary_sink);
-
- auto rtp_packet = CreateRtpPacketReceived();
- EXPECT_CALL(kept_secondary_sink, OnRtpPacket(SamePacketAs(*rtp_packet)));
-
- rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet);
-
- // Test tear-down.
- rtp_video_stream_receiver_->StopReceive();
- rtp_video_stream_receiver_->RemoveSecondarySink(&kept_secondary_sink);
-}
-
-TEST_F(RtpVideoStreamReceiverTest,
- SecondariesOfNonStartedStreamGetNoNotifications) {
- // Explicitly showing that the stream is not in the `started` state,
- // regardless of whether streams start out `started` or `stopped`.
- rtp_video_stream_receiver_->StopReceive();
-
- MockRtpPacketSink secondary_sink;
- rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink);
-
- auto rtp_packet = CreateRtpPacketReceived();
- EXPECT_CALL(secondary_sink, OnRtpPacket(_)).Times(0);
-
- rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet);
-
- // Test tear-down.
- rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink);
-}
-
-TEST_F(RtpVideoStreamReceiverTest, ParseGenericDescriptorOnePacket) {
- const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
- const int kSpatialIndex = 1;
-
- rtp_video_stream_receiver_->StartReceive();
-
- RtpHeaderExtensionMap extension_map;
- extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
- RtpPacketReceived rtp_packet(&extension_map);
- rtp_packet.SetPayloadType(kPayloadType);
-
- RtpGenericFrameDescriptor generic_descriptor;
- generic_descriptor.SetFirstPacketInSubFrame(true);
- generic_descriptor.SetLastPacketInSubFrame(true);
- generic_descriptor.SetFrameId(100);
- generic_descriptor.SetSpatialLayersBitmask(1 << kSpatialIndex);
- generic_descriptor.AddFrameDependencyDiff(90);
- generic_descriptor.AddFrameDependencyDiff(80);
- ASSERT_TRUE(rtp_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
- generic_descriptor));
-
- uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
- memcpy(payload, data.data(), data.size());
- // The first byte is the header, so we ignore the first byte of `data`.
- mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1,
- data.size() - 1);
-
- rtp_packet.SetMarker(true);
- rtp_packet.SetPayloadType(kPayloadType);
- rtp_packet.SetSequenceNumber(1);
-
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
- .WillOnce(Invoke([kSpatialIndex](EncodedFrame* frame) {
- EXPECT_EQ(frame->num_references, 2U);
- EXPECT_EQ(frame->references[0], frame->Id() - 90);
- EXPECT_EQ(frame->references[1], frame->Id() - 80);
- EXPECT_EQ(frame->SpatialIndex(), kSpatialIndex);
- EXPECT_THAT(frame->PacketInfos(), SizeIs(1));
- }));
-
- rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
-}
-
-TEST_F(RtpVideoStreamReceiverTest, ParseGenericDescriptorTwoPackets) {
- const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
- const int kSpatialIndex = 1;
-
- rtp_video_stream_receiver_->StartReceive();
-
- RtpHeaderExtensionMap extension_map;
- extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
- RtpPacketReceived first_packet(&extension_map);
-
- RtpGenericFrameDescriptor first_packet_descriptor;
- first_packet_descriptor.SetFirstPacketInSubFrame(true);
- first_packet_descriptor.SetLastPacketInSubFrame(false);
- first_packet_descriptor.SetFrameId(100);
- first_packet_descriptor.SetSpatialLayersBitmask(1 << kSpatialIndex);
- first_packet_descriptor.SetResolution(480, 360);
- ASSERT_TRUE(first_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
- first_packet_descriptor));
-
- uint8_t* first_packet_payload = first_packet.SetPayloadSize(data.size());
- memcpy(first_packet_payload, data.data(), data.size());
- // The first byte is the header, so we ignore the first byte of `data`.
- mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1,
- data.size() - 1);
-
- first_packet.SetPayloadType(kPayloadType);
- first_packet.SetSequenceNumber(1);
- rtp_video_stream_receiver_->OnRtpPacket(first_packet);
-
- RtpPacketReceived second_packet(&extension_map);
- RtpGenericFrameDescriptor second_packet_descriptor;
- second_packet_descriptor.SetFirstPacketInSubFrame(false);
- second_packet_descriptor.SetLastPacketInSubFrame(true);
- ASSERT_TRUE(second_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
- second_packet_descriptor));
-
- second_packet.SetMarker(true);
- second_packet.SetPayloadType(kPayloadType);
- second_packet.SetSequenceNumber(2);
-
- uint8_t* second_packet_payload = second_packet.SetPayloadSize(data.size());
- memcpy(second_packet_payload, data.data(), data.size());
- // The first byte is the header, so we ignore the first byte of `data`.
- mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1,
- data.size() - 1);
-
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
- .WillOnce(Invoke([kSpatialIndex](EncodedFrame* frame) {
- EXPECT_EQ(frame->num_references, 0U);
- EXPECT_EQ(frame->SpatialIndex(), kSpatialIndex);
- EXPECT_EQ(frame->EncodedImage()._encodedWidth, 480u);
- EXPECT_EQ(frame->EncodedImage()._encodedHeight, 360u);
- EXPECT_THAT(frame->PacketInfos(), SizeIs(2));
- }));
-
- rtp_video_stream_receiver_->OnRtpPacket(second_packet);
-}
-
-TEST_F(RtpVideoStreamReceiverTest, ParseGenericDescriptorRawPayload) {
- const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
- const int kRawPayloadType = 123;
-
- rtp_video_stream_receiver_->AddReceiveCodec(kRawPayloadType,
- kVideoCodecGeneric, {},
- /*raw_payload=*/true);
- rtp_video_stream_receiver_->StartReceive();
-
- RtpHeaderExtensionMap extension_map;
- extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
- RtpPacketReceived rtp_packet(&extension_map);
-
- RtpGenericFrameDescriptor generic_descriptor;
- generic_descriptor.SetFirstPacketInSubFrame(true);
- generic_descriptor.SetLastPacketInSubFrame(true);
- ASSERT_TRUE(rtp_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
- generic_descriptor));
-
- uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
- memcpy(payload, data.data(), data.size());
- mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
- data.size());
-
- rtp_packet.SetMarker(true);
- rtp_packet.SetPayloadType(kRawPayloadType);
- rtp_packet.SetSequenceNumber(1);
-
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame);
- rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
-}
-
-TEST_F(RtpVideoStreamReceiverTest, UnwrapsFrameId) {
- const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
- const int kPayloadType = 123;
-
- rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, kVideoCodecGeneric,
- {},
- /*raw_payload=*/true);
- rtp_video_stream_receiver_->StartReceive();
- RtpHeaderExtensionMap extension_map;
- extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
-
- uint16_t rtp_sequence_number = 1;
- auto inject_packet = [&](uint16_t wrapped_frame_id) {
- RtpPacketReceived rtp_packet(&extension_map);
-
- RtpGenericFrameDescriptor generic_descriptor;
- generic_descriptor.SetFirstPacketInSubFrame(true);
- generic_descriptor.SetLastPacketInSubFrame(true);
- generic_descriptor.SetFrameId(wrapped_frame_id);
- ASSERT_TRUE(rtp_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
- generic_descriptor));
-
- uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
- ASSERT_TRUE(payload);
- memcpy(payload, data.data(), data.size());
- mock_on_complete_frame_callback_.ClearExpectedBitstream();
- mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
- data.size());
- rtp_packet.SetMarker(true);
- rtp_packet.SetPayloadType(kPayloadType);
- rtp_packet.SetSequenceNumber(++rtp_sequence_number);
- rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
- };
-
- int64_t first_picture_id;
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
- .WillOnce([&](EncodedFrame* frame) { first_picture_id = frame->Id(); });
- inject_packet(/*wrapped_frame_id=*/0xffff);
-
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
- .WillOnce([&](EncodedFrame* frame) {
- EXPECT_EQ(frame->Id() - first_picture_id, 3);
- });
- inject_packet(/*wrapped_frame_id=*/0x0002);
-}
-
-class RtpVideoStreamReceiverDependencyDescriptorTest
- : public RtpVideoStreamReceiverTest {
- public:
- RtpVideoStreamReceiverDependencyDescriptorTest() {
- rtp_video_stream_receiver_->AddReceiveCodec(payload_type_,
- kVideoCodecGeneric, {},
- /*raw_payload=*/true);
- extension_map_.Register<RtpDependencyDescriptorExtension>(7);
- rtp_video_stream_receiver_->StartReceive();
- }
-
- // Returns some valid structure for the DependencyDescriptors.
- // First template of that structure always fit for a key frame.
- static FrameDependencyStructure CreateStreamStructure() {
- FrameDependencyStructure stream_structure;
- stream_structure.num_decode_targets = 1;
- stream_structure.templates = {
- FrameDependencyTemplate().Dtis("S"),
- FrameDependencyTemplate().Dtis("S").FrameDiffs({1}),
- };
- return stream_structure;
- }
-
- void InjectPacketWith(const FrameDependencyStructure& stream_structure,
- const DependencyDescriptor& dependency_descriptor) {
- const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
- RtpPacketReceived rtp_packet(&extension_map_);
- ASSERT_TRUE(rtp_packet.SetExtension<RtpDependencyDescriptorExtension>(
- stream_structure, dependency_descriptor));
- uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
- ASSERT_TRUE(payload);
- memcpy(payload, data.data(), data.size());
- mock_on_complete_frame_callback_.ClearExpectedBitstream();
- mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
- data.size());
- rtp_packet.SetMarker(true);
- rtp_packet.SetPayloadType(payload_type_);
- rtp_packet.SetSequenceNumber(++rtp_sequence_number_);
- rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
- }
-
- private:
- const int payload_type_ = 123;
- RtpHeaderExtensionMap extension_map_;
- uint16_t rtp_sequence_number_ = 321;
-};
-
-TEST_F(RtpVideoStreamReceiverDependencyDescriptorTest, UnwrapsFrameId) {
- FrameDependencyStructure stream_structure = CreateStreamStructure();
-
- DependencyDescriptor keyframe_descriptor;
- keyframe_descriptor.attached_structure =
- std::make_unique<FrameDependencyStructure>(stream_structure);
- keyframe_descriptor.frame_dependencies = stream_structure.templates[0];
- keyframe_descriptor.frame_number = 0xfff0;
- // DependencyDescriptor doesn't support reordering delta frame before
- // keyframe. Thus feed a key frame first, then test reodered delta frames.
- int64_t first_picture_id;
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
- .WillOnce([&](EncodedFrame* frame) { first_picture_id = frame->Id(); });
- InjectPacketWith(stream_structure, keyframe_descriptor);
-
- DependencyDescriptor deltaframe1_descriptor;
- deltaframe1_descriptor.frame_dependencies = stream_structure.templates[1];
- deltaframe1_descriptor.frame_number = 0xfffe;
-
- DependencyDescriptor deltaframe2_descriptor;
- deltaframe1_descriptor.frame_dependencies = stream_structure.templates[1];
- deltaframe2_descriptor.frame_number = 0x0002;
-
- // Parser should unwrap frame ids correctly even if packets were reordered by
- // the network.
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
- .WillOnce([&](EncodedFrame* frame) {
- // 0x0002 - 0xfff0
- EXPECT_EQ(frame->Id() - first_picture_id, 18);
- })
- .WillOnce([&](EncodedFrame* frame) {
- // 0xfffe - 0xfff0
- EXPECT_EQ(frame->Id() - first_picture_id, 14);
- });
- InjectPacketWith(stream_structure, deltaframe2_descriptor);
- InjectPacketWith(stream_structure, deltaframe1_descriptor);
-}
-
-TEST_F(RtpVideoStreamReceiverDependencyDescriptorTest,
- DropsLateDeltaFramePacketWithDependencyDescriptorExtension) {
- FrameDependencyStructure stream_structure1 = CreateStreamStructure();
- FrameDependencyStructure stream_structure2 = CreateStreamStructure();
- // Make sure template ids for these two structures do not collide:
- // adjust structure_id (that is also used as template id offset).
- stream_structure1.structure_id = 13;
- stream_structure2.structure_id =
- stream_structure1.structure_id + stream_structure1.templates.size();
-
- DependencyDescriptor keyframe1_descriptor;
- keyframe1_descriptor.attached_structure =
- std::make_unique<FrameDependencyStructure>(stream_structure1);
- keyframe1_descriptor.frame_dependencies = stream_structure1.templates[0];
- keyframe1_descriptor.frame_number = 1;
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame);
- InjectPacketWith(stream_structure1, keyframe1_descriptor);
-
- // Pass in 2nd key frame with different structure.
- DependencyDescriptor keyframe2_descriptor;
- keyframe2_descriptor.attached_structure =
- std::make_unique<FrameDependencyStructure>(stream_structure2);
- keyframe2_descriptor.frame_dependencies = stream_structure2.templates[0];
- keyframe2_descriptor.frame_number = 3;
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame);
- InjectPacketWith(stream_structure2, keyframe2_descriptor);
-
- // Pass in late delta frame that uses structure of the 1st key frame.
- DependencyDescriptor deltaframe_descriptor;
- deltaframe_descriptor.frame_dependencies = stream_structure1.templates[0];
- deltaframe_descriptor.frame_number = 2;
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame).Times(0);
- InjectPacketWith(stream_structure1, deltaframe_descriptor);
-}
-
-TEST_F(RtpVideoStreamReceiverDependencyDescriptorTest,
- DropsLateKeyFramePacketWithDependencyDescriptorExtension) {
- FrameDependencyStructure stream_structure1 = CreateStreamStructure();
- FrameDependencyStructure stream_structure2 = CreateStreamStructure();
- // Make sure template ids for these two structures do not collide:
- // adjust structure_id (that is also used as template id offset).
- stream_structure1.structure_id = 13;
- stream_structure2.structure_id =
- stream_structure1.structure_id + stream_structure1.templates.size();
-
- DependencyDescriptor keyframe1_descriptor;
- keyframe1_descriptor.attached_structure =
- std::make_unique<FrameDependencyStructure>(stream_structure1);
- keyframe1_descriptor.frame_dependencies = stream_structure1.templates[0];
- keyframe1_descriptor.frame_number = 1;
-
- DependencyDescriptor keyframe2_descriptor;
- keyframe2_descriptor.attached_structure =
- std::make_unique<FrameDependencyStructure>(stream_structure2);
- keyframe2_descriptor.frame_dependencies = stream_structure2.templates[0];
- keyframe2_descriptor.frame_number = 3;
-
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
- .WillOnce(
- [&](EncodedFrame* frame) { EXPECT_EQ(frame->Id() & 0xFFFF, 3); });
- InjectPacketWith(stream_structure2, keyframe2_descriptor);
- InjectPacketWith(stream_structure1, keyframe1_descriptor);
-
- // Pass in delta frame that uses structure of the 2nd key frame. Late key
- // frame shouldn't block it.
- DependencyDescriptor deltaframe_descriptor;
- deltaframe_descriptor.frame_dependencies = stream_structure2.templates[0];
- deltaframe_descriptor.frame_number = 4;
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
- .WillOnce(
- [&](EncodedFrame* frame) { EXPECT_EQ(frame->Id() & 0xFFFF, 4); });
- InjectPacketWith(stream_structure2, deltaframe_descriptor);
-}
-
-#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)
-using RtpVideoStreamReceiverDeathTest = RtpVideoStreamReceiverTest;
-TEST_F(RtpVideoStreamReceiverDeathTest, RepeatedSecondarySinkDisallowed) {
- MockRtpPacketSink secondary_sink;
-
- rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink);
- EXPECT_DEATH(rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink),
- "");
-
- // Test tear-down.
- rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink);
-}
-#endif
-
-TEST_F(RtpVideoStreamReceiverTest, TransformFrame) {
- auto mock_frame_transformer =
- rtc::make_ref_counted<testing::NiceMock<MockFrameTransformer>>();
- EXPECT_CALL(*mock_frame_transformer,
- RegisterTransformedFrameSinkCallback(_, config_.rtp.remote_ssrc));
- auto receiver = std::make_unique<RtpVideoStreamReceiver>(
- Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_,
- rtp_receive_statistics_.get(), nullptr, nullptr, process_thread_.get(),
- &mock_nack_sender_, nullptr, &mock_on_complete_frame_callback_, nullptr,
- mock_frame_transformer, &field_trials_);
- receiver->AddReceiveCodec(kPayloadType, kVideoCodecGeneric, {},
- /*raw_payload=*/false);
-
- RtpPacketReceived rtp_packet;
- rtp_packet.SetPayloadType(kPayloadType);
- rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'});
- rtp_packet.SetSequenceNumber(1);
- RTPVideoHeader video_header =
- GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
- mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
- data.size());
- EXPECT_CALL(*mock_frame_transformer, Transform(_));
- receiver->OnReceivedPayloadData(data, rtp_packet, video_header);
-
- EXPECT_CALL(*mock_frame_transformer,
- UnregisterTransformedFrameSinkCallback(config_.rtp.remote_ssrc));
- receiver = nullptr;
-}
-
-// Test default behavior and when playout delay is overridden by field trial.
-const VideoPlayoutDelay kTransmittedPlayoutDelay = {100, 200};
-const VideoPlayoutDelay kForcedPlayoutDelay = {70, 90};
-struct PlayoutDelayOptions {
- std::string field_trial;
- VideoPlayoutDelay expected_delay;
-};
-const PlayoutDelayOptions kDefaultBehavior = {
- /*field_trial=*/"", /*expected_delay=*/kTransmittedPlayoutDelay};
-const PlayoutDelayOptions kOverridePlayoutDelay = {
- /*field_trial=*/"WebRTC-ForcePlayoutDelay/min_ms:70,max_ms:90/",
- /*expected_delay=*/kForcedPlayoutDelay};
-
-class RtpVideoStreamReceiverTestPlayoutDelay
- : public RtpVideoStreamReceiverTest,
- public ::testing::WithParamInterface<PlayoutDelayOptions> {
- protected:
- RtpVideoStreamReceiverTestPlayoutDelay()
- : RtpVideoStreamReceiverTest(GetParam().field_trial) {}
-};
-
-INSTANTIATE_TEST_SUITE_P(PlayoutDelay,
- RtpVideoStreamReceiverTestPlayoutDelay,
- Values(kDefaultBehavior, kOverridePlayoutDelay));
-
-TEST_P(RtpVideoStreamReceiverTestPlayoutDelay, PlayoutDelay) {
- rtc::CopyOnWriteBuffer payload_data({'1', '2', '3', '4'});
- RtpHeaderExtensionMap extension_map;
- extension_map.Register<PlayoutDelayLimits>(1);
- RtpPacketToSend packet_to_send(&extension_map);
- packet_to_send.SetPayloadType(kPayloadType);
- packet_to_send.SetSequenceNumber(1);
-
- // Set playout delay on outgoing packet.
- EXPECT_TRUE(packet_to_send.SetExtension<PlayoutDelayLimits>(
- kTransmittedPlayoutDelay));
- uint8_t* payload = packet_to_send.AllocatePayload(payload_data.size());
- memcpy(payload, payload_data.data(), payload_data.size());
-
- RtpPacketReceived received_packet(&extension_map);
- received_packet.Parse(packet_to_send.data(), packet_to_send.size());
-
- RTPVideoHeader video_header =
- GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
- mock_on_complete_frame_callback_.AppendExpectedBitstream(payload_data.data(),
- payload_data.size());
- // Expect the playout delay of encoded frame to be the same as the transmitted
- // playout delay unless it was overridden by a field trial.
- EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
- .WillOnce(Invoke([expected_playout_delay =
- GetParam().expected_delay](EncodedFrame* frame) {
- EXPECT_EQ(frame->EncodedImage().playout_delay_, expected_playout_delay);
- }));
- rtp_video_stream_receiver_->OnReceivedPayloadData(
- received_packet.PayloadBuffer(), received_packet, video_header);
-}
-
-} // namespace webrtc
