Add flag enabling more packets to be retransmittable.
If not indicated otherwise, allow adding a packet to the retransmission
history at least every 1/15s in order to reduce frame dropping.
BUG=webrtc:7694
Review-Url: https://codereview.webrtc.org/2999063002
Cr-Original-Commit-Position: refs/heads/master@{#19665}
Cr-Mirrored-From: https://chromium.googlesource.com/external/webrtc
Cr-Mirrored-Commit: a8ae6f2acaef86e63f349442dbde8e81a9bd041e
diff --git a/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/modules/rtp_rtcp/include/rtp_rtcp_defines.h
index 7d54ea4..8add0a6 100644
--- a/modules/rtp_rtcp/include/rtp_rtcp_defines.h
+++ b/modules/rtp_rtcp/include/rtp_rtcp_defines.h
@@ -113,11 +113,16 @@
enum RtpRtcpPacketType { kPacketRtp = 0, kPacketKeepAlive = 1 };
+// kConditionallyRetransmitHigherLayers allows retransmission of video frames
+// in higher layers if either the last frame in that layer was too far back in
+// time, or if we estimate that a new frame will be available in a lower layer
+// in a shorter time than it would take to request and receive a retransmission.
enum RetransmissionMode : uint8_t {
kRetransmitOff = 0x0,
kRetransmitFECPackets = 0x1,
kRetransmitBaseLayer = 0x2,
kRetransmitHigherLayers = 0x4,
+ kConditionallyRetransmitHigherLayers = 0x8,
kRetransmitAllPackets = 0xFF
};
diff --git a/modules/rtp_rtcp/source/rtp_format.h b/modules/rtp_rtcp/source/rtp_format.h
index 378f38a..7b47921 100644
--- a/modules/rtp_rtcp/source/rtp_format.h
+++ b/modules/rtp_rtcp/source/rtp_format.h
@@ -41,10 +41,6 @@
// Returns true on success, false otherwise.
virtual bool NextPacket(RtpPacketToSend* packet) = 0;
- virtual ProtectionType GetProtectionType() = 0;
-
- virtual StorageType GetStorageType(uint32_t retransmission_settings) = 0;
-
virtual std::string ToString() = 0;
};
diff --git a/modules/rtp_rtcp/source/rtp_format_h264.cc b/modules/rtp_rtcp/source/rtp_format_h264.cc
index 092ac1c..5aa7e27 100644
--- a/modules/rtp_rtcp/source/rtp_format_h264.cc
+++ b/modules/rtp_rtcp/source/rtp_format_h264.cc
@@ -389,15 +389,6 @@
packets_.pop();
}
-ProtectionType RtpPacketizerH264::GetProtectionType() {
- return kProtectedPacket;
-}
-
-StorageType RtpPacketizerH264::GetStorageType(
- uint32_t retransmission_settings) {
- return kAllowRetransmission;
-}
-
std::string RtpPacketizerH264::ToString() {
return "RtpPacketizerH264";
}
diff --git a/modules/rtp_rtcp/source/rtp_format_h264.h b/modules/rtp_rtcp/source/rtp_format_h264.h
index 3742ec5..efa9b97 100644
--- a/modules/rtp_rtcp/source/rtp_format_h264.h
+++ b/modules/rtp_rtcp/source/rtp_format_h264.h
@@ -41,10 +41,6 @@
// Returns true on success, false otherwise.
bool NextPacket(RtpPacketToSend* rtp_packet) override;
- ProtectionType GetProtectionType() override;
-
- StorageType GetStorageType(uint32_t retransmission_settings) override;
-
std::string ToString() override;
private:
diff --git a/modules/rtp_rtcp/source/rtp_format_video_generic.cc b/modules/rtp_rtcp/source/rtp_format_video_generic.cc
index 696dc2b..ffb5f77 100644
--- a/modules/rtp_rtcp/source/rtp_format_video_generic.cc
+++ b/modules/rtp_rtcp/source/rtp_format_video_generic.cc
@@ -108,15 +108,6 @@
return true;
}
-ProtectionType RtpPacketizerGeneric::GetProtectionType() {
- return kProtectedPacket;
-}
-
-StorageType RtpPacketizerGeneric::GetStorageType(
- uint32_t retransmission_settings) {
- return kAllowRetransmission;
-}
-
std::string RtpPacketizerGeneric::ToString() {
return "RtpPacketizerGeneric";
}
diff --git a/modules/rtp_rtcp/source/rtp_format_video_generic.h b/modules/rtp_rtcp/source/rtp_format_video_generic.h
index 16d92e7..9fa3cce 100644
--- a/modules/rtp_rtcp/source/rtp_format_video_generic.h
+++ b/modules/rtp_rtcp/source/rtp_format_video_generic.h
@@ -43,10 +43,6 @@
// Returns true on success, false otherwise.
bool NextPacket(RtpPacketToSend* packet) override;
- ProtectionType GetProtectionType() override;
-
- StorageType GetStorageType(uint32_t retransmission_settings) override;
-
std::string ToString() override;
private:
diff --git a/modules/rtp_rtcp/source/rtp_format_vp8.cc b/modules/rtp_rtcp/source/rtp_format_vp8.cc
index b31de93..e56a977 100644
--- a/modules/rtp_rtcp/source/rtp_format_vp8.cc
+++ b/modules/rtp_rtcp/source/rtp_format_vp8.cc
@@ -211,25 +211,6 @@
return true;
}
-ProtectionType RtpPacketizerVp8::GetProtectionType() {
- bool protect =
- hdr_info_.temporalIdx == 0 || hdr_info_.temporalIdx == kNoTemporalIdx;
- return protect ? kProtectedPacket : kUnprotectedPacket;
-}
-
-StorageType RtpPacketizerVp8::GetStorageType(uint32_t retransmission_settings) {
- if (hdr_info_.temporalIdx == 0 &&
- !(retransmission_settings & kRetransmitBaseLayer)) {
- return kDontRetransmit;
- }
- if (hdr_info_.temporalIdx != kNoTemporalIdx &&
- hdr_info_.temporalIdx > 0 &&
- !(retransmission_settings & kRetransmitHigherLayers)) {
- return kDontRetransmit;
- }
- return kAllowRetransmission;
-}
-
std::string RtpPacketizerVp8::ToString() {
return "RtpPacketizerVp8";
}
diff --git a/modules/rtp_rtcp/source/rtp_format_vp8.h b/modules/rtp_rtcp/source/rtp_format_vp8.h
index edb3d4d..f8b7c2f 100644
--- a/modules/rtp_rtcp/source/rtp_format_vp8.h
+++ b/modules/rtp_rtcp/source/rtp_format_vp8.h
@@ -72,10 +72,6 @@
// Returns true on success, false otherwise.
bool NextPacket(RtpPacketToSend* packet) override;
- ProtectionType GetProtectionType() override;
-
- StorageType GetStorageType(uint32_t retransmission_settings) override;
-
std::string ToString() override;
private:
diff --git a/modules/rtp_rtcp/source/rtp_format_vp9.cc b/modules/rtp_rtcp/source/rtp_format_vp9.cc
index 6f02010..b08ec8b 100644
--- a/modules/rtp_rtcp/source/rtp_format_vp9.cc
+++ b/modules/rtp_rtcp/source/rtp_format_vp9.cc
@@ -472,24 +472,6 @@
RtpPacketizerVp9::~RtpPacketizerVp9() {
}
-ProtectionType RtpPacketizerVp9::GetProtectionType() {
- bool protect =
- hdr_.temporal_idx == 0 || hdr_.temporal_idx == kNoTemporalIdx;
- return protect ? kProtectedPacket : kUnprotectedPacket;
-}
-
-StorageType RtpPacketizerVp9::GetStorageType(uint32_t retransmission_settings) {
- StorageType storage = kAllowRetransmission;
- if (hdr_.temporal_idx == 0 &&
- !(retransmission_settings & kRetransmitBaseLayer)) {
- storage = kDontRetransmit;
- } else if (hdr_.temporal_idx != kNoTemporalIdx && hdr_.temporal_idx > 0 &&
- !(retransmission_settings & kRetransmitHigherLayers)) {
- storage = kDontRetransmit;
- }
- return storage;
-}
-
std::string RtpPacketizerVp9::ToString() {
return "RtpPacketizerVp9";
}
diff --git a/modules/rtp_rtcp/source/rtp_format_vp9.h b/modules/rtp_rtcp/source/rtp_format_vp9.h
index 7b0b06c..cf911a3 100644
--- a/modules/rtp_rtcp/source/rtp_format_vp9.h
+++ b/modules/rtp_rtcp/source/rtp_format_vp9.h
@@ -39,10 +39,6 @@
virtual ~RtpPacketizerVp9();
- ProtectionType GetProtectionType() override;
-
- StorageType GetStorageType(uint32_t retransmission_settings) override;
-
std::string ToString() override;
// The payload data must be one encoded VP9 layer frame.
diff --git a/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc b/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc
index 40d25d0..49ded07 100644
--- a/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc
+++ b/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc
@@ -580,37 +580,6 @@
EXPECT_TRUE(packet.Marker());
}
-TEST_F(RtpPacketizerVp9Test, TestBaseLayerProtectionAndStorageType) {
- const size_t kFrameSize = 10;
- const size_t kPacketSize = 12;
-
- // I:0, P:0, L:1, F:1, B:1, E:1, V:0 (2hdr + 10 payload)
- // L: T:0, U:0, S:0, D:0
- expected_.flexible_mode = true;
- expected_.temporal_idx = 0;
- Init(kFrameSize, kPacketSize);
- EXPECT_EQ(kProtectedPacket, packetizer_->GetProtectionType());
- EXPECT_EQ(kAllowRetransmission,
- packetizer_->GetStorageType(kRetransmitBaseLayer));
- EXPECT_EQ(kDontRetransmit, packetizer_->GetStorageType(kRetransmitOff));
-}
-
-TEST_F(RtpPacketizerVp9Test, TestHigherLayerProtectionAndStorageType) {
- const size_t kFrameSize = 10;
- const size_t kPacketSize = 12;
-
- // I:0, P:0, L:1, F:1, B:1, E:1, V:0 (2hdr + 10 payload)
- // L: T:1, U:0, S:0, D:0
- expected_.flexible_mode = true;
- expected_.temporal_idx = 1;
- Init(kFrameSize, kPacketSize);
- EXPECT_EQ(kUnprotectedPacket, packetizer_->GetProtectionType());
- EXPECT_EQ(kDontRetransmit, packetizer_->GetStorageType(kRetransmitBaseLayer));
- EXPECT_EQ(kAllowRetransmission,
- packetizer_->GetStorageType(kRetransmitHigherLayers));
-}
-
-
class RtpDepacketizerVp9Test : public ::testing::Test {
protected:
RtpDepacketizerVp9Test()
diff --git a/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/modules/rtp_rtcp/source/rtp_rtcp_impl.cc
index 31d1b0e..6167b44 100644
--- a/modules/rtp_rtcp/source/rtp_rtcp_impl.cc
+++ b/modules/rtp_rtcp/source/rtp_rtcp_impl.cc
@@ -31,6 +31,7 @@
const int64_t kRtpRtcpMaxIdleTimeProcessMs = 5;
const int64_t kRtpRtcpRttProcessTimeMs = 1000;
const int64_t kRtpRtcpBitrateProcessTimeMs = 10;
+const int64_t kDefaultExpectedRetransmissionTimeMs = 125;
} // namespace
RTPExtensionType StringToRtpExtensionType(const std::string& extension) {
@@ -430,9 +431,20 @@
if (rtcp_sender_.TimeToSendRTCPReport(kVideoFrameKey == frame_type)) {
rtcp_sender_.SendRTCP(GetFeedbackState(), kRtcpReport);
}
+ int64_t expected_retransmission_time_ms = rtt_ms();
+ if (expected_retransmission_time_ms == 0) {
+ // No rtt available (|kRtpRtcpRttProcessTimeMs| not yet passed?), so try to
+ // poll avg_rtt_ms directly from rtcp receiver.
+ if (rtcp_receiver_.RTT(rtcp_receiver_.RemoteSSRC(), nullptr,
+ &expected_retransmission_time_ms, nullptr,
+ nullptr) == -1) {
+ expected_retransmission_time_ms = kDefaultExpectedRetransmissionTimeMs;
+ }
+ }
return rtp_sender_->SendOutgoingData(
frame_type, payload_type, time_stamp, capture_time_ms, payload_data,
- payload_size, fragmentation, rtp_video_header, transport_frame_id_out);
+ payload_size, fragmentation, rtp_video_header, transport_frame_id_out,
+ expected_retransmission_time_ms);
}
bool ModuleRtpRtcpImpl::TimeToSendPacket(uint32_t ssrc,
diff --git a/modules/rtp_rtcp/source/rtp_sender.cc b/modules/rtp_rtcp/source/rtp_sender.cc
index 89e7735..8608759 100644
--- a/modules/rtp_rtcp/source/rtp_sender.cc
+++ b/modules/rtp_rtcp/source/rtp_sender.cc
@@ -232,7 +232,7 @@
if (payload_type_map_.end() != it) {
// We already use this payload type.
RtpUtility::Payload* payload = it->second;
- assert(payload);
+ RTC_DCHECK(payload);
// Check if it's the same as we already have.
if (RtpUtility::StringCompare(
@@ -355,7 +355,7 @@
}
SetSendPayloadType(payload_type);
RtpUtility::Payload* payload = it->second;
- assert(payload);
+ RTC_DCHECK(payload);
if (!payload->audio && !audio_configured_) {
video_->SetVideoCodecType(payload->typeSpecific.Video.videoCodecType);
*video_type = payload->typeSpecific.Video.videoCodecType;
@@ -371,7 +371,8 @@
size_t payload_size,
const RTPFragmentationHeader* fragmentation,
const RTPVideoHeader* rtp_header,
- uint32_t* transport_frame_id_out) {
+ uint32_t* transport_frame_id_out,
+ int64_t expected_retransmission_time_ms) {
uint32_t ssrc;
uint16_t sequence_number;
uint32_t rtp_timestamp;
@@ -395,20 +396,29 @@
return false;
}
+ switch (frame_type) {
+ case kAudioFrameSpeech:
+ case kAudioFrameCN:
+ RTC_CHECK(audio_configured_);
+ break;
+ case kVideoFrameKey:
+ case kVideoFrameDelta:
+ RTC_CHECK(!audio_configured_);
+ break;
+ case kEmptyFrame:
+ break;
+ }
+
bool result;
if (audio_configured_) {
TRACE_EVENT_ASYNC_STEP1("webrtc", "Audio", rtp_timestamp, "Send", "type",
FrameTypeToString(frame_type));
- assert(frame_type == kAudioFrameSpeech || frame_type == kAudioFrameCN ||
- frame_type == kEmptyFrame);
result = audio_->SendAudio(frame_type, payload_type, rtp_timestamp,
payload_data, payload_size, fragmentation);
} else {
TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", capture_time_ms,
"Send", "type", FrameTypeToString(frame_type));
- assert(frame_type != kAudioFrameSpeech && frame_type != kAudioFrameCN);
-
if (frame_type == kEmptyFrame)
return true;
@@ -419,7 +429,8 @@
result = video_->SendVideo(video_type, frame_type, payload_type,
rtp_timestamp, capture_time_ms, payload_data,
- payload_size, fragmentation, rtp_header);
+ payload_size, fragmentation, rtp_header,
+ expected_retransmission_time_ms);
}
rtc::CritScope cs(&statistics_crit_);
@@ -1105,7 +1116,7 @@
}
void RTPSender::SetCsrcs(const std::vector<uint32_t>& csrcs) {
- assert(csrcs.size() <= kRtpCsrcSize);
+ RTC_DCHECK_LE(csrcs.size(), kRtpCsrcSize);
rtc::CritScope lock(&send_critsect_);
csrcs_ = csrcs;
}
@@ -1136,7 +1147,7 @@
}
RtpVideoCodecTypes RTPSender::VideoCodecType() const {
- assert(!audio_configured_ && "Sender is an audio stream!");
+ RTC_DCHECK(!audio_configured_) << "Sender is an audio stream!";
return video_->VideoCodecType();
}
diff --git a/modules/rtp_rtcp/source/rtp_sender.h b/modules/rtp_rtcp/source/rtp_sender.h
index 069357f..9c4fdc1 100644
--- a/modules/rtp_rtcp/source/rtp_sender.h
+++ b/modules/rtp_rtcp/source/rtp_sender.h
@@ -108,7 +108,8 @@
size_t payload_size,
const RTPFragmentationHeader* fragmentation,
const RTPVideoHeader* rtp_header,
- uint32_t* transport_frame_id_out);
+ uint32_t* transport_frame_id_out,
+ int64_t expected_retransmission_time_ms);
// RTP header extension
int32_t RegisterRtpHeaderExtension(RTPExtensionType type, uint8_t id);
diff --git a/modules/rtp_rtcp/source/rtp_sender_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_unittest.cc
index 14bbff7..5a47759 100644
--- a/modules/rtp_rtcp/source/rtp_sender_unittest.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_unittest.cc
@@ -25,7 +25,9 @@
#include "webrtc/modules/rtp_rtcp/source/rtp_sender.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_sender_video.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_utility.h"
+#include "webrtc/rtc_base/arraysize.h"
#include "webrtc/rtc_base/buffer.h"
+#include "webrtc/rtc_base/ptr_util.h"
#include "webrtc/rtc_base/rate_limiter.h"
#include "webrtc/test/field_trial.h"
#include "webrtc/test/gmock.h"
@@ -55,6 +57,7 @@
const int kVideoRotationExtensionId = 5;
const size_t kGenericHeaderLength = 1;
const uint8_t kPayloadData[] = {47, 11, 32, 93, 89};
+const int64_t kDefaultExpectedRetransmissionTimeMs = 125;
using ::testing::_;
using ::testing::ElementsAreArray;
@@ -238,7 +241,8 @@
EXPECT_TRUE(rtp_sender_->SendOutgoingData(
kVideoFrameKey, kPayloadType, kTimestamp, kCaptureTimeMs, kPayloadData,
- sizeof(kPayloadData), nullptr, nullptr, nullptr));
+ sizeof(kPayloadData), nullptr, nullptr, nullptr,
+ kDefaultExpectedRetransmissionTimeMs));
}
};
@@ -249,15 +253,32 @@
void SetUp() override { SetUpRtpSender(false); }
};
+class TestRtpSenderVideo : public RTPSenderVideo {
+ public:
+ TestRtpSenderVideo(Clock* clock,
+ RTPSender* rtp_sender,
+ FlexfecSender* flexfec_sender)
+ : RTPSenderVideo(clock, rtp_sender, flexfec_sender) {}
+ ~TestRtpSenderVideo() override {}
+
+ StorageType GetStorageType(const RTPVideoHeader& header,
+ int32_t retransmission_settings,
+ int64_t expected_retransmission_time_ms) {
+ return RTPSenderVideo::GetStorageType(GetTemporalId(header),
+ retransmission_settings,
+ expected_retransmission_time_ms);
+ }
+};
+
class RtpSenderVideoTest : public RtpSenderTest {
protected:
void SetUp() override {
// TODO(pbos): Set up to use pacer.
SetUpRtpSender(false);
rtp_sender_video_.reset(
- new RTPSenderVideo(&fake_clock_, rtp_sender_.get(), nullptr));
+ new TestRtpSenderVideo(&fake_clock_, rtp_sender_.get(), nullptr));
}
- std::unique_ptr<RTPSenderVideo> rtp_sender_video_;
+ std::unique_ptr<TestRtpSenderVideo> rtp_sender_video_;
};
TEST_P(RtpSenderTestWithoutPacer, AllocatePacketSetCsrc) {
@@ -861,9 +882,9 @@
uint8_t payload[] = {47, 11, 32, 93, 89};
// Send keyframe
- ASSERT_TRUE(rtp_sender_->SendOutgoingData(kVideoFrameKey, payload_type, 1234,
- 4321, payload, sizeof(payload),
- nullptr, nullptr, nullptr));
+ ASSERT_TRUE(rtp_sender_->SendOutgoingData(
+ kVideoFrameKey, payload_type, 1234, 4321, payload, sizeof(payload),
+ nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs));
auto sent_payload = transport_.last_sent_packet().payload();
uint8_t generic_header = sent_payload[0];
@@ -878,7 +899,7 @@
ASSERT_TRUE(rtp_sender_->SendOutgoingData(
kVideoFrameDelta, payload_type, 1234, 4321, payload, sizeof(payload),
- nullptr, nullptr, nullptr));
+ nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs));
sent_payload = transport_.last_sent_packet().payload();
generic_header = sent_payload[0];
@@ -998,7 +1019,8 @@
video_header.video_timing.flags = TimingFrameFlags::kTriggeredByTimer;
EXPECT_TRUE(rtp_sender_->SendOutgoingData(
kVideoFrameKey, kPayloadType, kTimestamp, kCaptureTimeMs, kPayloadData,
- sizeof(kPayloadData), nullptr, &video_header, nullptr));
+ sizeof(kPayloadData), nullptr, &video_header, nullptr,
+ kDefaultExpectedRetransmissionTimeMs));
EXPECT_CALL(mock_rtc_event_log_,
LogRtpHeader(PacketDirection::kOutgoingPacket, _, _, _))
@@ -1023,7 +1045,8 @@
video_header.video_timing.flags = TimingFrameFlags::kInvalid;
EXPECT_TRUE(rtp_sender_->SendOutgoingData(
kVideoFrameKey, kPayloadType, kTimestamp + 1, kCaptureTimeMs + 1,
- kPayloadData, sizeof(kPayloadData), nullptr, &video_header, nullptr));
+ kPayloadData, sizeof(kPayloadData), nullptr, &video_header, nullptr,
+ kDefaultExpectedRetransmissionTimeMs));
EXPECT_CALL(mock_rtc_event_log_,
LogRtpHeader(PacketDirection::kOutgoingPacket, _, _, _))
@@ -1168,9 +1191,9 @@
EXPECT_CALL(mock_paced_sender_, InsertPacket(_, _, _, _, _, _))
.Times(::testing::AtLeast(2));
- ASSERT_TRUE(rtp_sender_->SendOutgoingData(kVideoFrameKey, payload_type, 1234,
- 4321, payload, sizeof(payload),
- nullptr, nullptr, nullptr));
+ ASSERT_TRUE(rtp_sender_->SendOutgoingData(
+ kVideoFrameKey, payload_type, 1234, 4321, payload, sizeof(payload),
+ nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs));
EXPECT_EQ(1U, callback.num_calls_);
EXPECT_EQ(ssrc, callback.ssrc_);
@@ -1179,7 +1202,7 @@
ASSERT_TRUE(rtp_sender_->SendOutgoingData(
kVideoFrameDelta, payload_type, 1234, 4321, payload, sizeof(payload),
- nullptr, nullptr, nullptr));
+ nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs));
EXPECT_EQ(2U, callback.num_calls_);
EXPECT_EQ(ssrc, callback.ssrc_);
@@ -1245,7 +1268,7 @@
for (uint32_t i = 0; i < kNumPackets; ++i) {
ASSERT_TRUE(rtp_sender_->SendOutgoingData(
kVideoFrameKey, payload_type, 1234, 4321, payload, sizeof(payload),
- nullptr, nullptr, nullptr));
+ nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs));
fake_clock_.AdvanceTimeMilliseconds(kPacketInterval);
}
@@ -1327,8 +1350,8 @@
// Send a frame.
ASSERT_TRUE(rtp_sender_->SendOutgoingData(
- kVideoFrameKey, payload_type, 1234, 4321, payload,
- sizeof(payload), nullptr, nullptr, nullptr));
+ kVideoFrameKey, payload_type, 1234, 4321, payload, sizeof(payload),
+ nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs));
StreamDataCounters expected;
expected.transmitted.payload_bytes = 6;
expected.transmitted.header_bytes = 12;
@@ -1369,8 +1392,8 @@
fec_params.max_fec_frames = 1;
rtp_sender_->SetFecParameters(fec_params, fec_params);
ASSERT_TRUE(rtp_sender_->SendOutgoingData(
- kVideoFrameDelta, payload_type, 1234, 4321, payload,
- sizeof(payload), nullptr, nullptr, nullptr));
+ kVideoFrameDelta, payload_type, 1234, 4321, payload, sizeof(payload),
+ nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs));
expected.transmitted.payload_bytes = 40;
expected.transmitted.header_bytes = 60;
expected.transmitted.packets = 5;
@@ -1388,8 +1411,8 @@
uint8_t payload[] = {47, 11, 32, 93, 89};
ASSERT_TRUE(rtp_sender_->SendOutgoingData(
- kAudioFrameCN, payload_type, 1234, 4321, payload,
- sizeof(payload), nullptr, nullptr, nullptr));
+ kAudioFrameCN, payload_type, 1234, 4321, payload, sizeof(payload),
+ nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs));
auto sent_payload = transport_.last_sent_packet().payload();
EXPECT_THAT(sent_payload, ElementsAreArray(payload));
@@ -1407,8 +1430,8 @@
uint8_t payload[] = {47, 11, 32, 93, 89};
ASSERT_TRUE(rtp_sender_->SendOutgoingData(
- kAudioFrameCN, payload_type, 1234, 4321, payload,
- sizeof(payload), nullptr, nullptr, nullptr));
+ kAudioFrameCN, payload_type, 1234, 4321, payload, sizeof(payload),
+ nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs));
auto sent_payload = transport_.last_sent_packet().payload();
EXPECT_THAT(sent_payload, ElementsAreArray(payload));
@@ -1445,22 +1468,22 @@
// During start, it takes the starting timestamp as last sent timestamp.
// The duration is calculated as the difference of current and last sent
// timestamp. So for first call it will skip since the duration is zero.
- ASSERT_TRUE(rtp_sender_->SendOutgoingData(kEmptyFrame, kPayloadType,
- capture_time_ms, 0, nullptr, 0,
- nullptr, nullptr, nullptr));
+ ASSERT_TRUE(rtp_sender_->SendOutgoingData(
+ kEmptyFrame, kPayloadType, capture_time_ms, 0, nullptr, 0, nullptr,
+ nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs));
// DTMF Sample Length is (Frequency/1000) * Duration.
// So in this case, it is (8000/1000) * 500 = 4000.
// Sending it as two packets.
ASSERT_TRUE(rtp_sender_->SendOutgoingData(
- kEmptyFrame, kPayloadType, capture_time_ms + 2000, 0,
- nullptr, 0, nullptr, nullptr, nullptr));
+ kEmptyFrame, kPayloadType, capture_time_ms + 2000, 0, nullptr, 0, nullptr,
+ nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs));
// Marker Bit should be set to 1 for first packet.
EXPECT_TRUE(transport_.last_sent_packet().Marker());
ASSERT_TRUE(rtp_sender_->SendOutgoingData(
- kEmptyFrame, kPayloadType, capture_time_ms + 4000, 0,
- nullptr, 0, nullptr, nullptr, nullptr));
+ kEmptyFrame, kPayloadType, capture_time_ms + 4000, 0, nullptr, 0, nullptr,
+ nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs));
// Marker Bit should be set to 0 for rest of the packets.
EXPECT_FALSE(transport_.last_sent_packet().Marker());
}
@@ -1478,8 +1501,8 @@
uint8_t payload[] = {47, 11, 32, 93, 89};
ASSERT_TRUE(rtp_sender_->SendOutgoingData(
- kVideoFrameKey, kPayloadType, 1234, 4321, payload,
- sizeof(payload), nullptr, nullptr, nullptr));
+ kVideoFrameKey, kPayloadType, 1234, 4321, payload, sizeof(payload),
+ nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs));
// Will send 2 full-size padding packets.
rtp_sender_->TimeToSendPadding(1, PacedPacketInfo());
@@ -1553,7 +1576,7 @@
hdr.rotation = kVideoRotation_0;
rtp_sender_video_->SendVideo(kRtpVideoGeneric, kVideoFrameKey, kPayload,
kTimestamp, 0, kFrame, sizeof(kFrame), nullptr,
- &hdr);
+ &hdr, kDefaultExpectedRetransmissionTimeMs);
VideoRotation rotation;
EXPECT_TRUE(
@@ -1579,7 +1602,8 @@
fake_clock_.AdvanceTimeMilliseconds(kPacketizationTimeMs);
rtp_sender_video_->SendVideo(kRtpVideoGeneric, kVideoFrameKey, kPayload,
kTimestamp, kCaptureTimestamp, kFrame,
- sizeof(kFrame), nullptr, &hdr);
+ sizeof(kFrame), nullptr, &hdr,
+ kDefaultExpectedRetransmissionTimeMs);
VideoSendTiming timing;
EXPECT_TRUE(transport_.last_sent_packet().GetExtension<VideoTimingExtension>(
&timing));
@@ -1595,14 +1619,14 @@
RTPVideoHeader hdr = {0};
hdr.rotation = kVideoRotation_90;
- EXPECT_TRUE(rtp_sender_video_->SendVideo(kRtpVideoGeneric, kVideoFrameKey,
- kPayload, kTimestamp, 0, kFrame,
- sizeof(kFrame), nullptr, &hdr));
+ EXPECT_TRUE(rtp_sender_video_->SendVideo(
+ kRtpVideoGeneric, kVideoFrameKey, kPayload, kTimestamp, 0, kFrame,
+ sizeof(kFrame), nullptr, &hdr, kDefaultExpectedRetransmissionTimeMs));
hdr.rotation = kVideoRotation_0;
- EXPECT_TRUE(rtp_sender_video_->SendVideo(kRtpVideoGeneric, kVideoFrameDelta,
- kPayload, kTimestamp + 1, 0, kFrame,
- sizeof(kFrame), nullptr, &hdr));
+ EXPECT_TRUE(rtp_sender_video_->SendVideo(
+ kRtpVideoGeneric, kVideoFrameDelta, kPayload, kTimestamp + 1, 0, kFrame,
+ sizeof(kFrame), nullptr, &hdr, kDefaultExpectedRetransmissionTimeMs));
VideoRotation rotation;
EXPECT_TRUE(
@@ -1617,13 +1641,13 @@
RTPVideoHeader hdr = {0};
hdr.rotation = kVideoRotation_90;
- EXPECT_TRUE(rtp_sender_video_->SendVideo(kRtpVideoGeneric, kVideoFrameKey,
- kPayload, kTimestamp, 0, kFrame,
- sizeof(kFrame), nullptr, &hdr));
+ EXPECT_TRUE(rtp_sender_video_->SendVideo(
+ kRtpVideoGeneric, kVideoFrameKey, kPayload, kTimestamp, 0, kFrame,
+ sizeof(kFrame), nullptr, &hdr, kDefaultExpectedRetransmissionTimeMs));
- EXPECT_TRUE(rtp_sender_video_->SendVideo(kRtpVideoGeneric, kVideoFrameDelta,
- kPayload, kTimestamp + 1, 0, kFrame,
- sizeof(kFrame), nullptr, &hdr));
+ EXPECT_TRUE(rtp_sender_video_->SendVideo(
+ kRtpVideoGeneric, kVideoFrameDelta, kPayload, kTimestamp + 1, 0, kFrame,
+ sizeof(kFrame), nullptr, &hdr, kDefaultExpectedRetransmissionTimeMs));
VideoRotation rotation;
EXPECT_TRUE(
@@ -1652,6 +1676,224 @@
ConvertCVOByteToVideoRotation(flip_bit | camera_bit | 3));
}
+TEST_P(RtpSenderVideoTest, RetransmissionTypesGeneric) {
+ RTPVideoHeader header;
+ header.codec = kRtpVideoGeneric;
+
+ EXPECT_EQ(kDontRetransmit,
+ rtp_sender_video_->GetStorageType(
+ header, kRetransmitOff, kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType(
+ header, kRetransmitBaseLayer,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType(
+ header, kRetransmitHigherLayers,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission,
+ rtp_sender_video_->GetStorageType(
+ header, kConditionallyRetransmitHigherLayers,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType(
+ header, kRetransmitAllPackets,
+ kDefaultExpectedRetransmissionTimeMs));
+}
+
+TEST_P(RtpSenderVideoTest, RetransmissionTypesH264) {
+ RTPVideoHeader header;
+ header.codec = kRtpVideoH264;
+ header.codecHeader.H264.packetization_mode =
+ H264PacketizationMode::NonInterleaved;
+
+ EXPECT_EQ(kDontRetransmit,
+ rtp_sender_video_->GetStorageType(
+ header, kRetransmitOff, kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType(
+ header, kRetransmitBaseLayer,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType(
+ header, kRetransmitHigherLayers,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission,
+ rtp_sender_video_->GetStorageType(
+ header, kConditionallyRetransmitHigherLayers,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType(
+ header, kRetransmitAllPackets,
+ kDefaultExpectedRetransmissionTimeMs));
+}
+
+TEST_P(RtpSenderVideoTest, RetransmissionTypesVP8BaseLayer) {
+ RTPVideoHeader header;
+ header.codec = kRtpVideoVp8;
+ header.codecHeader.VP8.temporalIdx = 0;
+
+ EXPECT_EQ(kDontRetransmit,
+ rtp_sender_video_->GetStorageType(
+ header, kRetransmitOff, kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType(
+ header, kRetransmitBaseLayer,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kDontRetransmit, rtp_sender_video_->GetStorageType(
+ header, kRetransmitHigherLayers,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission,
+ rtp_sender_video_->GetStorageType(
+ header, kRetransmitHigherLayers | kRetransmitBaseLayer,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kDontRetransmit, rtp_sender_video_->GetStorageType(
+ header, kConditionallyRetransmitHigherLayers,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(
+ kAllowRetransmission,
+ rtp_sender_video_->GetStorageType(
+ header, kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType(
+ header, kRetransmitAllPackets,
+ kDefaultExpectedRetransmissionTimeMs));
+}
+
+TEST_P(RtpSenderVideoTest, RetransmissionTypesVP8HigherLayers) {
+ RTPVideoHeader header;
+ header.codec = kRtpVideoVp8;
+
+ for (int tid = 1; tid <= kMaxTemporalStreams; ++tid) {
+ header.codecHeader.VP8.temporalIdx = tid;
+
+ EXPECT_EQ(kDontRetransmit, rtp_sender_video_->GetStorageType(
+ header, kRetransmitOff,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kDontRetransmit, rtp_sender_video_->GetStorageType(
+ header, kRetransmitBaseLayer,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType(
+ header, kRetransmitHigherLayers,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission,
+ rtp_sender_video_->GetStorageType(
+ header, kRetransmitHigherLayers | kRetransmitBaseLayer,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType(
+ header, kRetransmitAllPackets,
+ kDefaultExpectedRetransmissionTimeMs));
+ }
+}
+
+TEST_P(RtpSenderVideoTest, RetransmissionTypesVP9) {
+ RTPVideoHeader header;
+ header.codec = kRtpVideoVp9;
+
+ for (int tid = 1; tid <= kMaxTemporalStreams; ++tid) {
+ header.codecHeader.VP9.temporal_idx = tid;
+
+ EXPECT_EQ(kDontRetransmit, rtp_sender_video_->GetStorageType(
+ header, kRetransmitOff,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kDontRetransmit, rtp_sender_video_->GetStorageType(
+ header, kRetransmitBaseLayer,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType(
+ header, kRetransmitHigherLayers,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission,
+ rtp_sender_video_->GetStorageType(
+ header, kRetransmitHigherLayers | kRetransmitBaseLayer,
+ kDefaultExpectedRetransmissionTimeMs));
+ EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType(
+ header, kRetransmitAllPackets,
+ kDefaultExpectedRetransmissionTimeMs));
+ }
+}
+
+TEST_P(RtpSenderVideoTest, ConditionalRetransmit) {
+ const int64_t kFrameIntervalMs = 33;
+ const int64_t kRttMs = (kFrameIntervalMs * 3) / 2;
+ const uint8_t kSettings =
+ kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers;
+
+ // Insert VP8 frames for all temporal layers, but stop before the final index.
+ RTPVideoHeader header;
+ header.codec = kRtpVideoVp8;
+
+ // Fill averaging window to prevent rounding errors.
+ constexpr int kNumRepetitions =
+ (RTPSenderVideo::kTLRateWindowSizeMs + (kFrameIntervalMs / 2)) /
+ kFrameIntervalMs;
+ constexpr int kPattern[] = {0, 2, 1, 2};
+ for (size_t i = 0; i < arraysize(kPattern) * kNumRepetitions; ++i) {
+ header.codecHeader.VP8.temporalIdx = kPattern[i % arraysize(kPattern)];
+ rtp_sender_video_->GetStorageType(header, kSettings, kRttMs);
+ fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
+ }
+
+ // Since we're at the start of the pattern, the next expected frame in TL0 is
+ // right now. We will wait at most one expected retransmission time before
+ // acknowledging that it did not arrive, which means this frame and the next
+ // will not be retransmitted.
+ header.codecHeader.VP8.temporalIdx = 1;
+ EXPECT_EQ(StorageType::kDontRetransmit,
+ rtp_sender_video_->GetStorageType(header, kSettings, kRttMs));
+ fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
+ EXPECT_EQ(StorageType::kDontRetransmit,
+ rtp_sender_video_->GetStorageType(header, kSettings, kRttMs));
+ fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
+
+ // The TL0 frame did not arrive. So allow retransmission.
+ EXPECT_EQ(StorageType::kAllowRetransmission,
+ rtp_sender_video_->GetStorageType(header, kSettings, kRttMs));
+ fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
+
+ // Insert a frame for TL2. We just had frame in TL1, so the next one there is
+ // in three frames away. TL0 is still too far in the past. So, allow
+ // retransmission.
+ header.codecHeader.VP8.temporalIdx = 2;
+ EXPECT_EQ(StorageType::kAllowRetransmission,
+ rtp_sender_video_->GetStorageType(header, kSettings, kRttMs));
+ fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
+
+ // Another TL2, next in TL1 is two frames away. Allow again.
+ EXPECT_EQ(StorageType::kAllowRetransmission,
+ rtp_sender_video_->GetStorageType(header, kSettings, kRttMs));
+ fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
+
+ // Yet another TL2, next in TL1 is now only one frame away, so don't store
+ // for retransmission.
+ EXPECT_EQ(StorageType::kDontRetransmit,
+ rtp_sender_video_->GetStorageType(header, kSettings, kRttMs));
+}
+
+TEST_P(RtpSenderVideoTest, ConditionalRetransmitLimit) {
+ const int64_t kFrameIntervalMs = 200;
+ const int64_t kRttMs = (kFrameIntervalMs * 3) / 2;
+ const int32_t kSettings =
+ kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers;
+
+ // Insert VP8 frames for all temporal layers, but stop before the final index.
+ RTPVideoHeader header;
+ header.codec = kRtpVideoVp8;
+
+ // Fill averaging window to prevent rounding errors.
+ constexpr int kNumRepetitions =
+ (RTPSenderVideo::kTLRateWindowSizeMs + (kFrameIntervalMs / 2)) /
+ kFrameIntervalMs;
+ constexpr int kPattern[] = {0, 2, 2, 2};
+ for (size_t i = 0; i < arraysize(kPattern) * kNumRepetitions; ++i) {
+ header.codecHeader.VP8.temporalIdx = kPattern[i % arraysize(kPattern)];
+
+ rtp_sender_video_->GetStorageType(header, kSettings, kRttMs);
+ fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
+ }
+
+ // Since we're at the start of the pattern, the next expected frame will be
+ // right now in TL0. Put it in TL1 instead. Regular rules would dictate that
+ // we don't store for retransmission because we expect a frame in a lower
+ // layer, but that last frame in TL1 was a long time ago in absolute terms,
+ // so allow retransmission anyway.
+ header.codecHeader.VP8.temporalIdx = 1;
+ EXPECT_EQ(StorageType::kAllowRetransmission,
+ rtp_sender_video_->GetStorageType(header, kSettings, kRttMs));
+}
+
TEST_P(RtpSenderTest, OnOverheadChanged) {
MockOverheadObserver mock_overhead_observer;
rtp_sender_.reset(
diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc
index 3845074..4e46b78 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video.cc
@@ -13,9 +13,10 @@
#include <stdlib.h>
#include <string.h>
+#include <limits>
#include <memory>
-#include <vector>
#include <utility>
+#include <vector>
#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "webrtc/modules/rtp_rtcp/source/byte_io.h"
@@ -33,6 +34,7 @@
namespace {
constexpr size_t kRedForFecHeaderLength = 1;
+constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4;
void BuildRedPayload(const RtpPacketToSend& media_packet,
RtpPacketToSend* red_packet) {
@@ -53,7 +55,8 @@
: rtp_sender_(rtp_sender),
clock_(clock),
video_type_(kRtpVideoGeneric),
- retransmission_settings_(kRetransmitBaseLayer),
+ retransmission_settings_(kRetransmitBaseLayer |
+ kConditionallyRetransmitHigherLayers),
last_rotation_(kVideoRotation_0),
red_payload_type_(-1),
ulpfec_payload_type_(-1),
@@ -292,7 +295,8 @@
const uint8_t* payload_data,
size_t payload_size,
const RTPFragmentationHeader* fragmentation,
- const RTPVideoHeader* video_header) {
+ const RTPVideoHeader* video_header,
+ int64_t expected_retransmission_time_ms) {
if (payload_size == 0)
return false;
@@ -365,8 +369,11 @@
std::unique_ptr<RtpPacketizer> packetizer(RtpPacketizer::Create(
video_type, max_data_payload_length, last_packet_reduction_len,
video_header ? &(video_header->codecHeader) : nullptr, frame_type));
- // Media packet storage.
- StorageType storage = packetizer->GetStorageType(retransmission_settings);
+
+ const uint8_t temporal_id =
+ video_header ? GetTemporalId(*video_header) : kNoTemporalIdx;
+ StorageType storage = GetStorageType(temporal_id, retransmission_settings,
+ expected_retransmission_time_ms);
// TODO(changbin): we currently don't support to configure the codec to
// output multiple partitions for VP8. Should remove below check after the
@@ -392,7 +399,9 @@
if (!rtp_sender_->AssignSequenceNumber(packet.get()))
return false;
- bool protect_packet = (packetizer->GetProtectionType() == kProtectedPacket);
+ // No FEC protection for upper temporal layers, if used.
+ bool protect_packet = temporal_id == 0 || temporal_id == kNoTemporalIdx;
+
// Put packetization finish timestamp into extension.
if (packet->HasExtension<VideoTimingExtension>()) {
packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds());
@@ -453,4 +462,90 @@
retransmission_settings_ = settings;
}
+StorageType RTPSenderVideo::GetStorageType(
+ uint8_t temporal_id,
+ int32_t retransmission_settings,
+ int64_t expected_retransmission_time_ms) {
+ if (retransmission_settings == kRetransmitOff)
+ return StorageType::kDontRetransmit;
+ if (retransmission_settings == kRetransmitAllPackets)
+ return StorageType::kAllowRetransmission;
+
+ rtc::CritScope cs(&stats_crit_);
+ // Media packet storage.
+ if ((retransmission_settings & kConditionallyRetransmitHigherLayers) &&
+ UpdateConditionalRetransmit(temporal_id,
+ expected_retransmission_time_ms)) {
+ retransmission_settings |= kRetransmitHigherLayers;
+ }
+
+ if (temporal_id == kNoTemporalIdx)
+ return kAllowRetransmission;
+
+ if ((retransmission_settings & kRetransmitBaseLayer) && temporal_id == 0)
+ return kAllowRetransmission;
+
+ if ((retransmission_settings & kRetransmitHigherLayers) && temporal_id > 0)
+ return kAllowRetransmission;
+
+ return kDontRetransmit;
+}
+
+uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) {
+ switch (header.codec) {
+ case kRtpVideoVp8:
+ return header.codecHeader.VP8.temporalIdx;
+ case kRtpVideoVp9:
+ return header.codecHeader.VP9.temporal_idx;
+ default:
+ return kNoTemporalIdx;
+ }
+}
+
+bool RTPSenderVideo::UpdateConditionalRetransmit(
+ uint8_t temporal_id,
+ int64_t expected_retransmission_time_ms) {
+ int64_t now_ms = clock_->TimeInMilliseconds();
+ // Update stats for any temporal layer.
+ TemporalLayerStats* current_layer_stats =
+ &frame_stats_by_temporal_layer_[temporal_id];
+ current_layer_stats->frame_rate_fp1000s.Update(1, now_ms);
+ int64_t tl_frame_interval = now_ms - current_layer_stats->last_frame_time_ms;
+ current_layer_stats->last_frame_time_ms = now_ms;
+
+ // Conditional retransmit only applies to upper layers.
+ if (temporal_id != kNoTemporalIdx && temporal_id > 0) {
+ if (tl_frame_interval >= kMaxUnretransmittableFrameIntervalMs) {
+ // Too long since a retransmittable frame in this layer, enable NACK
+ // protection.
+ return true;
+ } else {
+ // Estimate when the next frame of any lower layer will be sent.
+ const int64_t kUndefined = std::numeric_limits<int64_t>::max();
+ int64_t expected_next_frame_time = kUndefined;
+ for (int i = temporal_id - 1; i >= 0; --i) {
+ TemporalLayerStats* stats = &frame_stats_by_temporal_layer_[i];
+ rtc::Optional<uint32_t> rate = stats->frame_rate_fp1000s.Rate(now_ms);
+ if (rate) {
+ int64_t tl_next = stats->last_frame_time_ms + 1000000 / *rate;
+ if (tl_next - now_ms > -expected_retransmission_time_ms &&
+ tl_next < expected_next_frame_time) {
+ expected_next_frame_time = tl_next;
+ }
+ }
+ }
+
+ if (expected_next_frame_time == kUndefined ||
+ expected_next_frame_time - now_ms > expected_retransmission_time_ms) {
+ // The next frame in a lower layer is expected at a later time (or
+ // unable to tell due to lack of data) than a retransmission is
+ // estimated to be able to arrive, so allow this packet to be nacked.
+ return true;
+ }
+ }
+ }
+
+ return false;
+}
+
} // namespace webrtc
diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h
index 9fb4648..e4a2405 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video.h
+++ b/modules/rtp_rtcp/source/rtp_sender_video.h
@@ -11,9 +11,8 @@
#ifndef WEBRTC_MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_
#define WEBRTC_MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_
-#include <list>
+#include <map>
#include <memory>
-#include <vector>
#include "webrtc/common_types.h"
#include "webrtc/modules/rtp_rtcp/include/flexfec_sender.h"
@@ -32,10 +31,13 @@
#include "webrtc/typedefs.h"
namespace webrtc {
+class RtpPacketizer;
class RtpPacketToSend;
class RTPSenderVideo {
public:
+ static constexpr int64_t kTLRateWindowSizeMs = 2500;
+
RTPSenderVideo(Clock* clock,
RTPSender* rtpSender,
FlexfecSender* flexfec_sender);
@@ -55,7 +57,8 @@
const uint8_t* payload_data,
size_t payload_size,
const RTPFragmentationHeader* fragmentation,
- const RTPVideoHeader* video_header);
+ const RTPVideoHeader* video_header,
+ int64_t expected_retransmission_time_ms);
void SetVideoCodecType(RtpVideoCodecTypes type);
@@ -76,7 +79,24 @@
int SelectiveRetransmissions() const;
void SetSelectiveRetransmissions(uint8_t settings);
+ protected:
+ static uint8_t GetTemporalId(const RTPVideoHeader& header);
+ StorageType GetStorageType(uint8_t temporal_id,
+ int32_t retransmission_settings,
+ int64_t expected_retransmission_time_ms);
+
private:
+ struct TemporalLayerStats {
+ TemporalLayerStats()
+ : frame_rate_fp1000s(kTLRateWindowSizeMs, 1000 * 1000),
+ last_frame_time_ms(0) {}
+ // Frame rate, in frames per 1000 seconds. This essentially turns the fps
+ // value into a fixed point value with three decimals. Improves precision at
+ // low frame rates.
+ RateStatistics frame_rate_fp1000s;
+ int64_t last_frame_time_ms;
+ };
+
size_t CalculateFecPacketOverhead() const EXCLUSIVE_LOCKS_REQUIRED(crit_);
void SendVideoPacket(std::unique_ptr<RtpPacketToSend> packet,
@@ -103,6 +123,10 @@
bool flexfec_enabled() const { return flexfec_sender_ != nullptr; }
+ bool UpdateConditionalRetransmit(uint8_t temporal_id,
+ int64_t expected_retransmission_time_ms)
+ EXCLUSIVE_LOCKS_REQUIRED(stats_crit_);
+
RTPSender* const rtp_sender_;
Clock* const clock_;
@@ -131,6 +155,10 @@
RateStatistics fec_bitrate_ GUARDED_BY(stats_crit_);
// Bitrate used for video payload and RTP headers.
RateStatistics video_bitrate_ GUARDED_BY(stats_crit_);
+
+ std::map<int, TemporalLayerStats> frame_stats_by_temporal_layer_
+ GUARDED_BY(stats_crit_);
+
OneTimeEvent first_frame_sent_;
};
