Give VideoSendStreamImpl access to RTP timestamps
When a LossNotification RTCP message is received, the sequence numbers
it refers to must be converted to timestamps before passing the message
down to the encoder. This CL gives VideoSendStreamImpl access to that
information via VideoSendStreamImpl::rtp_video_sender_.
TBR=sprang@webrtc.org
Bug: webrtc:10501
Change-Id: If207f0b6d2fb344da35b525cc104e8ba5cc614ec
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/131323
Commit-Queue: Elad Alon <eladalon@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27489}diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc
index 74015a8..cd9283e 100644
--- a/call/rtp_video_sender.cc
+++ b/call/rtp_video_sender.cc
@@ -740,6 +740,17 @@
return protection_bitrate_bps_;
}
+absl::optional<RtpSequenceNumberMap::Info> RtpVideoSender::GetSentRtpPacketInfo(
+ uint32_t ssrc,
+ uint16_t seq_num) const {
+ for (const auto& rtp_stream : rtp_streams_) {
+ if (ssrc == rtp_stream.rtp_rtcp->SSRC()) {
+ return rtp_stream.sender_video->GetSentRtpPacketInfo(seq_num);
+ }
+ }
+ return absl::nullopt;
+}
+
int RtpVideoSender::ProtectionRequest(const FecProtectionParams* delta_params,
const FecProtectionParams* key_params,
uint32_t* sent_video_rate_bps,
diff --git a/call/rtp_video_sender.h b/call/rtp_video_sender.h
index d50cb7c..8518c91 100644
--- a/call/rtp_video_sender.h
+++ b/call/rtp_video_sender.h
@@ -16,6 +16,7 @@
#include <unordered_set>
#include <vector>
+#include "absl/types/optional.h"
#include "api/call/transport.h"
#include "api/fec_controller.h"
#include "api/video_codecs/video_encoder.h"
@@ -26,6 +27,7 @@
#include "logging/rtc_event_log/rtc_event_log.h"
#include "modules/rtp_rtcp/include/flexfec_sender.h"
#include "modules/rtp_rtcp/source/rtp_sender_video.h"
+#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h"
#include "modules/rtp_rtcp/source/rtp_video_header.h"
#include "modules/utility/include/process_thread.h"
#include "rtc_base/constructor_magic.h"
@@ -138,6 +140,10 @@
size_t height,
size_t num_temporal_layers) override;
+ absl::optional<RtpSequenceNumberMap::Info> GetSentRtpPacketInfo(
+ uint32_t ssrc,
+ uint16_t seq_num) const override;
+
// From PacketFeedbackObserver.
void OnPacketAdded(uint32_t ssrc, uint16_t seq_num) override;
void OnPacketFeedbackVector(
diff --git a/call/rtp_video_sender_interface.h b/call/rtp_video_sender_interface.h
index ecaca9b..51cf56b 100644
--- a/call/rtp_video_sender_interface.h
+++ b/call/rtp_video_sender_interface.h
@@ -14,8 +14,10 @@
#include <map>
#include <vector>
+#include "absl/types/optional.h"
#include "call/rtp_config.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
+#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h"
#include "modules/utility/include/process_thread.h"
#include "modules/video_coding/include/video_codec_interface.h"
@@ -55,6 +57,9 @@
virtual void SetEncodingData(size_t width,
size_t height,
size_t num_temporal_layers) = 0;
+ virtual absl::optional<RtpSequenceNumberMap::Info> GetSentRtpPacketInfo(
+ uint32_t ssrc,
+ uint16_t seq_num) const = 0;
};
} // namespace webrtc
#endif // CALL_RTP_VIDEO_SENDER_INTERFACE_H_
diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc
index 10bc218..a90afbf 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video.cc
@@ -15,6 +15,7 @@
#include <limits>
#include <memory>
+#include <string>
#include <utility>
#include <vector>
@@ -38,6 +39,7 @@
namespace {
constexpr size_t kRedForFecHeaderLength = 1;
+constexpr size_t kRtpSequenceNumberMapMaxEntries = 1 << 13;
constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4;
void BuildRedPayload(const RtpPacketToSend& media_packet,
@@ -196,6 +198,14 @@
last_rotation_(kVideoRotation_0),
transmit_color_space_next_frame_(false),
playout_delay_oracle_(playout_delay_oracle),
+ // TODO(eladalon): Choose whether to instantiate rtp_sequence_number_map_
+ // according to the negotiation of the RTCP message.
+ rtp_sequence_number_map_(
+ field_trials.Lookup("WebRTC-RtcpLossNotification").find("Enabled") !=
+ std::string::npos
+ ? absl::make_unique<RtpSequenceNumberMap>(
+ kRtpSequenceNumberMapMaxEntries)
+ : nullptr),
red_payload_type_(-1),
ulpfec_payload_type_(-1),
flexfec_sender_(flexfec_sender),
@@ -622,7 +632,7 @@
const uint8_t temporal_id = GetTemporalId(*video_header);
StorageType storage = GetStorageType(temporal_id, retransmission_settings,
expected_retransmission_time_ms);
- size_t num_packets = packetizer->NumPackets();
+ const size_t num_packets = packetizer->NumPackets();
size_t unpacketized_payload_size;
if (fragmentation && fragmentation->fragmentationVectorSize > 0) {
@@ -638,6 +648,7 @@
if (num_packets == 0)
return false;
+ uint16_t first_sequence_number;
bool first_frame = first_frame_sent_();
for (size_t i = 0; i < num_packets; ++i) {
std::unique_ptr<RtpPacketToSend> packet;
@@ -667,6 +678,10 @@
return false;
packetized_payload_size += packet->payload_size();
+ if (rtp_sequence_number_map_ && i == 0) {
+ first_sequence_number = packet->SequenceNumber();
+ }
+
if (i == 0) {
playout_delay_oracle_->OnSentPacket(packet->SequenceNumber(),
playout_delay);
@@ -709,6 +724,13 @@
}
}
+ if (rtp_sequence_number_map_) {
+ const uint32_t timestamp = rtp_timestamp - rtp_sender_->TimestampOffset();
+ rtc::CritScope cs(&crit_);
+ rtp_sequence_number_map_->InsertFrame(first_sequence_number, num_packets,
+ timestamp);
+ }
+
rtc::CritScope cs(&stats_crit_);
RTC_DCHECK_GE(packetized_payload_size, unpacketized_payload_size);
packetization_overhead_bitrate_.Update(
@@ -736,6 +758,15 @@
.value_or(0);
}
+absl::optional<RtpSequenceNumberMap::Info> RTPSenderVideo::GetSentRtpPacketInfo(
+ uint16_t sequence_number) const {
+ if (!rtp_sequence_number_map_) {
+ return absl::nullopt;
+ }
+ rtc::CritScope cs(&crit_);
+ return rtp_sequence_number_map_->Get(sequence_number);
+}
+
StorageType RTPSenderVideo::GetStorageType(
uint8_t temporal_id,
int32_t retransmission_settings,
diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h
index 034e790..7144afd 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video.h
+++ b/modules/rtp_rtcp/source/rtp_sender_video.h
@@ -21,6 +21,7 @@
#include "modules/rtp_rtcp/source/playout_delay_oracle.h"
#include "modules/rtp_rtcp/source/rtp_rtcp_config.h"
#include "modules/rtp_rtcp/source/rtp_sender.h"
+#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h"
#include "modules/rtp_rtcp/source/ulpfec_generator.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/one_time_event.h"
@@ -93,6 +94,14 @@
// or extension/
uint32_t PacketizationOverheadBps() const;
+ // Recall the last RTP packet whose sequence number was |sequence_number|.
+ // Return the timestamp of the video frame that packet belonged too, as well
+ // as whether the packet was the first and/or last packet in the frame.
+ // absl::nullopt returned if no such packet can be recalled (e.g. it happened
+ // too long ago).
+ absl::optional<RtpSequenceNumberMap::Info> GetSentRtpPacketInfo(
+ uint16_t sequence_number) const;
+
protected:
static uint8_t GetTemporalId(const RTPVideoHeader& header);
StorageType GetStorageType(uint8_t temporal_id,
@@ -166,6 +175,13 @@
// delay extension on header.
PlayoutDelayOracle* const playout_delay_oracle_;
+ // Maps sent packets' sequence numbers to a tuple consisting of:
+ // 1. The timestamp, without the randomizing offset mandated by the RFC.
+ // 2. Whether the packet was the first in its frame.
+ // 3. Whether the packet was the last in its frame.
+ const std::unique_ptr<RtpSequenceNumberMap> rtp_sequence_number_map_
+ RTC_PT_GUARDED_BY(crit_);
+
// RED/ULPFEC.
int red_payload_type_ RTC_GUARDED_BY(crit_);
int ulpfec_payload_type_ RTC_GUARDED_BY(crit_);
diff --git a/video/video_send_stream_impl_unittest.cc b/video/video_send_stream_impl_unittest.cc
index cc45476..fc0f785 100644
--- a/video/video_send_stream_impl_unittest.cc
+++ b/video/video_send_stream_impl_unittest.cc
@@ -11,10 +11,12 @@
#include <string>
#include "absl/memory/memory.h"
+#include "absl/types/optional.h"
#include "call/rtp_video_sender.h"
#include "call/test/mock_bitrate_allocator.h"
#include "call/test/mock_rtp_transport_controller_send.h"
#include "logging/rtc_event_log/rtc_event_log.h"
+#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h"
#include "modules/utility/include/process_thread.h"
#include "modules/video_coding/fec_controller_default.h"
#include "rtc_base/experiments/alr_experiment.h"
@@ -67,6 +69,10 @@
MOCK_CONST_METHOD0(GetPayloadBitrateBps, uint32_t());
MOCK_CONST_METHOD0(GetProtectionBitrateBps, uint32_t());
MOCK_METHOD3(SetEncodingData, void(size_t, size_t, size_t));
+ MOCK_CONST_METHOD2(
+ GetSentRtpPacketInfo,
+ absl::optional<RtpSequenceNumberMap::Info>(uint32_t ssrc,
+ uint16_t seq_num));
};
BitrateAllocationUpdate CreateAllocation(int bitrate_bps) {
