Fork NackModule and RtpVideoStreamReceiver
Bug: webrtc:11595
Change-Id: I4d14c0bf9c32e09d1624099a256f2778afebd4df
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/175901
Commit-Queue: Tommi <tommi@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31337}
diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index 028b0d5..b084577 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -74,6 +74,8 @@
"histogram.h",
"nack_module.cc",
"nack_module.h",
+ "nack_module2.cc",
+ "nack_module2.h",
]
deps = [
@@ -876,6 +878,7 @@
"jitter_buffer_unittest.cc",
"jitter_estimator_tests.cc",
"loss_notification_controller_unittest.cc",
+ "nack_module2_unittest.cc",
"nack_module_unittest.cc",
"packet_buffer_unittest.cc",
"receiver_unittest.cc",
diff --git a/modules/video_coding/nack_module2.cc b/modules/video_coding/nack_module2.cc
new file mode 100644
index 0000000..267eaeb
--- /dev/null
+++ b/modules/video_coding/nack_module2.cc
@@ -0,0 +1,349 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/nack_module2.h"
+
+#include <algorithm>
+#include <limits>
+
+#include "api/units/timestamp.h"
+#include "modules/utility/include/process_thread.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/experiments/field_trial_parser.h"
+#include "rtc_base/logging.h"
+#include "system_wrappers/include/field_trial.h"
+
+namespace webrtc {
+
+namespace {
+const int kMaxPacketAge = 10000;
+const int kMaxNackPackets = 1000;
+const int kDefaultRttMs = 100;
+const int kMaxNackRetries = 10;
+const int kProcessFrequency = 50;
+const int kProcessIntervalMs = 1000 / kProcessFrequency;
+const int kMaxReorderedPackets = 128;
+const int kNumReorderingBuckets = 10;
+const int kDefaultSendNackDelayMs = 0;
+
+int64_t GetSendNackDelay() {
+ int64_t delay_ms = strtol(
+ webrtc::field_trial::FindFullName("WebRTC-SendNackDelayMs").c_str(),
+ nullptr, 10);
+ if (delay_ms > 0 && delay_ms <= 20) {
+ RTC_LOG(LS_INFO) << "SendNackDelay is set to " << delay_ms;
+ return delay_ms;
+ }
+ return kDefaultSendNackDelayMs;
+}
+} // namespace
+
+NackModule2::NackInfo::NackInfo()
+ : seq_num(0), send_at_seq_num(0), sent_at_time(-1), retries(0) {}
+
+NackModule2::NackInfo::NackInfo(uint16_t seq_num,
+ uint16_t send_at_seq_num,
+ int64_t created_at_time)
+ : seq_num(seq_num),
+ send_at_seq_num(send_at_seq_num),
+ created_at_time(created_at_time),
+ sent_at_time(-1),
+ retries(0) {}
+
+NackModule2::BackoffSettings::BackoffSettings(TimeDelta min_retry,
+ TimeDelta max_rtt,
+ double base)
+ : min_retry_interval(min_retry), max_rtt(max_rtt), base(base) {}
+
+absl::optional<NackModule2::BackoffSettings>
+NackModule2::BackoffSettings::ParseFromFieldTrials() {
+ // Matches magic number in RTPSender::OnReceivedNack().
+ const TimeDelta kDefaultMinRetryInterval = TimeDelta::Millis(5);
+ // Upper bound on link-delay considered for exponential backoff.
+ // Selected so that cumulative delay with 1.25 base and 10 retries ends up
+ // below 3s, since above that there will be a FIR generated instead.
+ const TimeDelta kDefaultMaxRtt = TimeDelta::Millis(160);
+ // Default base for exponential backoff, adds 25% RTT delay for each retry.
+ const double kDefaultBase = 1.25;
+
+ FieldTrialParameter<bool> enabled("enabled", false);
+ FieldTrialParameter<TimeDelta> min_retry("min_retry",
+ kDefaultMinRetryInterval);
+ FieldTrialParameter<TimeDelta> max_rtt("max_rtt", kDefaultMaxRtt);
+ FieldTrialParameter<double> base("base", kDefaultBase);
+ ParseFieldTrial({&enabled, &min_retry, &max_rtt, &base},
+ field_trial::FindFullName("WebRTC-ExponentialNackBackoff"));
+
+ if (enabled) {
+ return NackModule2::BackoffSettings(min_retry.Get(), max_rtt.Get(),
+ base.Get());
+ }
+ return absl::nullopt;
+}
+
+NackModule2::NackModule2(Clock* clock,
+ NackSender* nack_sender,
+ KeyFrameRequestSender* keyframe_request_sender)
+ : clock_(clock),
+ nack_sender_(nack_sender),
+ keyframe_request_sender_(keyframe_request_sender),
+ reordering_histogram_(kNumReorderingBuckets, kMaxReorderedPackets),
+ initialized_(false),
+ rtt_ms_(kDefaultRttMs),
+ newest_seq_num_(0),
+ next_process_time_ms_(-1),
+ send_nack_delay_ms_(GetSendNackDelay()),
+ backoff_settings_(BackoffSettings::ParseFromFieldTrials()) {
+ RTC_DCHECK(clock_);
+ RTC_DCHECK(nack_sender_);
+ RTC_DCHECK(keyframe_request_sender_);
+}
+
+int NackModule2::OnReceivedPacket(uint16_t seq_num, bool is_keyframe) {
+ return OnReceivedPacket(seq_num, is_keyframe, false);
+}
+
+int NackModule2::OnReceivedPacket(uint16_t seq_num,
+ bool is_keyframe,
+ bool is_recovered) {
+ rtc::CritScope lock(&crit_);
+ // TODO(philipel): When the packet includes information whether it is
+ // retransmitted or not, use that value instead. For
+ // now set it to true, which will cause the reordering
+ // statistics to never be updated.
+ bool is_retransmitted = true;
+
+ if (!initialized_) {
+ newest_seq_num_ = seq_num;
+ if (is_keyframe)
+ keyframe_list_.insert(seq_num);
+ initialized_ = true;
+ return 0;
+ }
+
+ // Since the |newest_seq_num_| is a packet we have actually received we know
+ // that packet has never been Nacked.
+ if (seq_num == newest_seq_num_)
+ return 0;
+
+ if (AheadOf(newest_seq_num_, seq_num)) {
+ // An out of order packet has been received.
+ auto nack_list_it = nack_list_.find(seq_num);
+ int nacks_sent_for_packet = 0;
+ if (nack_list_it != nack_list_.end()) {
+ nacks_sent_for_packet = nack_list_it->second.retries;
+ nack_list_.erase(nack_list_it);
+ }
+ if (!is_retransmitted)
+ UpdateReorderingStatistics(seq_num);
+ return nacks_sent_for_packet;
+ }
+
+ // Keep track of new keyframes.
+ if (is_keyframe)
+ keyframe_list_.insert(seq_num);
+
+ // And remove old ones so we don't accumulate keyframes.
+ auto it = keyframe_list_.lower_bound(seq_num - kMaxPacketAge);
+ if (it != keyframe_list_.begin())
+ keyframe_list_.erase(keyframe_list_.begin(), it);
+
+ if (is_recovered) {
+ recovered_list_.insert(seq_num);
+
+ // Remove old ones so we don't accumulate recovered packets.
+ auto it = recovered_list_.lower_bound(seq_num - kMaxPacketAge);
+ if (it != recovered_list_.begin())
+ recovered_list_.erase(recovered_list_.begin(), it);
+
+ // Do not send nack for packets recovered by FEC or RTX.
+ return 0;
+ }
+
+ AddPacketsToNack(newest_seq_num_ + 1, seq_num);
+ newest_seq_num_ = seq_num;
+
+ // Are there any nacks that are waiting for this seq_num.
+ std::vector<uint16_t> nack_batch = GetNackBatch(kSeqNumOnly);
+ if (!nack_batch.empty()) {
+ // This batch of NACKs is triggered externally; the initiator can
+ // batch them with other feedback messages.
+ nack_sender_->SendNack(nack_batch, /*buffering_allowed=*/true);
+ }
+
+ return 0;
+}
+
+void NackModule2::ClearUpTo(uint16_t seq_num) {
+ rtc::CritScope lock(&crit_);
+ nack_list_.erase(nack_list_.begin(), nack_list_.lower_bound(seq_num));
+ keyframe_list_.erase(keyframe_list_.begin(),
+ keyframe_list_.lower_bound(seq_num));
+ recovered_list_.erase(recovered_list_.begin(),
+ recovered_list_.lower_bound(seq_num));
+}
+
+void NackModule2::UpdateRtt(int64_t rtt_ms) {
+ rtc::CritScope lock(&crit_);
+ rtt_ms_ = rtt_ms;
+}
+
+void NackModule2::Clear() {
+ rtc::CritScope lock(&crit_);
+ nack_list_.clear();
+ keyframe_list_.clear();
+ recovered_list_.clear();
+}
+
+int64_t NackModule2::TimeUntilNextProcess() {
+ return std::max<int64_t>(next_process_time_ms_ - clock_->TimeInMilliseconds(),
+ 0);
+}
+
+void NackModule2::Process() {
+ if (nack_sender_) {
+ std::vector<uint16_t> nack_batch;
+ {
+ rtc::CritScope lock(&crit_);
+ nack_batch = GetNackBatch(kTimeOnly);
+ }
+
+ if (!nack_batch.empty()) {
+ // This batch of NACKs is triggered externally; there is no external
+ // initiator who can batch them with other feedback messages.
+ nack_sender_->SendNack(nack_batch, /*buffering_allowed=*/false);
+ }
+ }
+
+ // Update the next_process_time_ms_ in intervals to achieve
+ // the targeted frequency over time. Also add multiple intervals
+ // in case of a skip in time as to not make uneccessary
+ // calls to Process in order to catch up.
+ int64_t now_ms = clock_->TimeInMilliseconds();
+ if (next_process_time_ms_ == -1) {
+ next_process_time_ms_ = now_ms + kProcessIntervalMs;
+ } else {
+ next_process_time_ms_ = next_process_time_ms_ + kProcessIntervalMs +
+ (now_ms - next_process_time_ms_) /
+ kProcessIntervalMs * kProcessIntervalMs;
+ }
+}
+
+bool NackModule2::RemovePacketsUntilKeyFrame() {
+ while (!keyframe_list_.empty()) {
+ auto it = nack_list_.lower_bound(*keyframe_list_.begin());
+
+ if (it != nack_list_.begin()) {
+ // We have found a keyframe that actually is newer than at least one
+ // packet in the nack list.
+ nack_list_.erase(nack_list_.begin(), it);
+ return true;
+ }
+
+ // If this keyframe is so old it does not remove any packets from the list,
+ // remove it from the list of keyframes and try the next keyframe.
+ keyframe_list_.erase(keyframe_list_.begin());
+ }
+ return false;
+}
+
+void NackModule2::AddPacketsToNack(uint16_t seq_num_start,
+ uint16_t seq_num_end) {
+ // Remove old packets.
+ auto it = nack_list_.lower_bound(seq_num_end - kMaxPacketAge);
+ nack_list_.erase(nack_list_.begin(), it);
+
+ // If the nack list is too large, remove packets from the nack list until
+ // the latest first packet of a keyframe. If the list is still too large,
+ // clear it and request a keyframe.
+ uint16_t num_new_nacks = ForwardDiff(seq_num_start, seq_num_end);
+ if (nack_list_.size() + num_new_nacks > kMaxNackPackets) {
+ while (RemovePacketsUntilKeyFrame() &&
+ nack_list_.size() + num_new_nacks > kMaxNackPackets) {
+ }
+
+ if (nack_list_.size() + num_new_nacks > kMaxNackPackets) {
+ nack_list_.clear();
+ RTC_LOG(LS_WARNING) << "NACK list full, clearing NACK"
+ " list and requesting keyframe.";
+ keyframe_request_sender_->RequestKeyFrame();
+ return;
+ }
+ }
+
+ for (uint16_t seq_num = seq_num_start; seq_num != seq_num_end; ++seq_num) {
+ // Do not send nack for packets that are already recovered by FEC or RTX
+ if (recovered_list_.find(seq_num) != recovered_list_.end())
+ continue;
+ NackInfo nack_info(seq_num, seq_num + WaitNumberOfPackets(0.5),
+ clock_->TimeInMilliseconds());
+ RTC_DCHECK(nack_list_.find(seq_num) == nack_list_.end());
+ nack_list_[seq_num] = nack_info;
+ }
+}
+
+std::vector<uint16_t> NackModule2::GetNackBatch(NackFilterOptions options) {
+ bool consider_seq_num = options != kTimeOnly;
+ bool consider_timestamp = options != kSeqNumOnly;
+ Timestamp now = clock_->CurrentTime();
+ std::vector<uint16_t> nack_batch;
+ auto it = nack_list_.begin();
+ while (it != nack_list_.end()) {
+ TimeDelta resend_delay = TimeDelta::Millis(rtt_ms_);
+ if (backoff_settings_) {
+ resend_delay =
+ std::max(resend_delay, backoff_settings_->min_retry_interval);
+ if (it->second.retries > 1) {
+ TimeDelta exponential_backoff =
+ std::min(TimeDelta::Millis(rtt_ms_), backoff_settings_->max_rtt) *
+ std::pow(backoff_settings_->base, it->second.retries - 1);
+ resend_delay = std::max(resend_delay, exponential_backoff);
+ }
+ }
+
+ bool delay_timed_out =
+ now.ms() - it->second.created_at_time >= send_nack_delay_ms_;
+ bool nack_on_rtt_passed =
+ now.ms() - it->second.sent_at_time >= resend_delay.ms();
+ bool nack_on_seq_num_passed =
+ it->second.sent_at_time == -1 &&
+ AheadOrAt(newest_seq_num_, it->second.send_at_seq_num);
+ if (delay_timed_out && ((consider_seq_num && nack_on_seq_num_passed) ||
+ (consider_timestamp && nack_on_rtt_passed))) {
+ nack_batch.emplace_back(it->second.seq_num);
+ ++it->second.retries;
+ it->second.sent_at_time = now.ms();
+ if (it->second.retries >= kMaxNackRetries) {
+ RTC_LOG(LS_WARNING) << "Sequence number " << it->second.seq_num
+ << " removed from NACK list due to max retries.";
+ it = nack_list_.erase(it);
+ } else {
+ ++it;
+ }
+ continue;
+ }
+ ++it;
+ }
+ return nack_batch;
+}
+
+void NackModule2::UpdateReorderingStatistics(uint16_t seq_num) {
+ RTC_DCHECK(AheadOf(newest_seq_num_, seq_num));
+ uint16_t diff = ReverseDiff(newest_seq_num_, seq_num);
+ reordering_histogram_.Add(diff);
+}
+
+int NackModule2::WaitNumberOfPackets(float probability) const {
+ if (reordering_histogram_.NumValues() == 0)
+ return 0;
+ return reordering_histogram_.InverseCdf(probability);
+}
+
+} // namespace webrtc
diff --git a/modules/video_coding/nack_module2.h b/modules/video_coding/nack_module2.h
new file mode 100644
index 0000000..6518f32
--- /dev/null
+++ b/modules/video_coding/nack_module2.h
@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_NACK_MODULE2_H_
+#define MODULES_VIDEO_CODING_NACK_MODULE2_H_
+
+#include <stdint.h>
+
+#include <map>
+#include <set>
+#include <vector>
+
+#include "api/units/time_delta.h"
+#include "modules/include/module.h"
+#include "modules/include/module_common_types.h"
+#include "modules/video_coding/histogram.h"
+#include "rtc_base/critical_section.h"
+#include "rtc_base/numerics/sequence_number_util.h"
+#include "rtc_base/thread_annotations.h"
+#include "system_wrappers/include/clock.h"
+
+namespace webrtc {
+
+class NackModule2 final : public Module {
+ public:
+ NackModule2(Clock* clock,
+ NackSender* nack_sender,
+ KeyFrameRequestSender* keyframe_request_sender);
+
+ int OnReceivedPacket(uint16_t seq_num, bool is_keyframe);
+ int OnReceivedPacket(uint16_t seq_num, bool is_keyframe, bool is_recovered);
+
+ void ClearUpTo(uint16_t seq_num);
+ void UpdateRtt(int64_t rtt_ms);
+ void Clear();
+
+ // Module implementation
+ int64_t TimeUntilNextProcess() override;
+ void Process() override;
+
+ private:
+ // Which fields to consider when deciding which packet to nack in
+ // GetNackBatch.
+ enum NackFilterOptions { kSeqNumOnly, kTimeOnly, kSeqNumAndTime };
+
+ // This class holds the sequence number of the packet that is in the nack list
+ // as well as the meta data about when it should be nacked and how many times
+ // we have tried to nack this packet.
+ struct NackInfo {
+ NackInfo();
+ NackInfo(uint16_t seq_num,
+ uint16_t send_at_seq_num,
+ int64_t created_at_time);
+
+ uint16_t seq_num;
+ uint16_t send_at_seq_num;
+ int64_t created_at_time;
+ int64_t sent_at_time;
+ int retries;
+ };
+
+ struct BackoffSettings {
+ BackoffSettings(TimeDelta min_retry, TimeDelta max_rtt, double base);
+ static absl::optional<BackoffSettings> ParseFromFieldTrials();
+
+ // Min time between nacks.
+ const TimeDelta min_retry_interval;
+ // Upper bound on link-delay considered for exponential backoff.
+ const TimeDelta max_rtt;
+ // Base for the exponential backoff.
+ const double base;
+ };
+
+ void AddPacketsToNack(uint16_t seq_num_start, uint16_t seq_num_end)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
+
+ // Removes packets from the nack list until the next keyframe. Returns true
+ // if packets were removed.
+ bool RemovePacketsUntilKeyFrame() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
+ std::vector<uint16_t> GetNackBatch(NackFilterOptions options)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
+
+ // Update the reordering distribution.
+ void UpdateReorderingStatistics(uint16_t seq_num)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
+
+ // Returns how many packets we have to wait in order to receive the packet
+ // with probability |probabilty| or higher.
+ int WaitNumberOfPackets(float probability) const
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
+
+ rtc::CriticalSection crit_;
+ Clock* const clock_;
+ NackSender* const nack_sender_;
+ KeyFrameRequestSender* const keyframe_request_sender_;
+
+ // TODO(philipel): Some of the variables below are consistently used on a
+ // known thread (e.g. see |initialized_|). Those probably do not need
+ // synchronized access.
+ std::map<uint16_t, NackInfo, DescendingSeqNumComp<uint16_t>> nack_list_
+ RTC_GUARDED_BY(crit_);
+ std::set<uint16_t, DescendingSeqNumComp<uint16_t>> keyframe_list_
+ RTC_GUARDED_BY(crit_);
+ std::set<uint16_t, DescendingSeqNumComp<uint16_t>> recovered_list_
+ RTC_GUARDED_BY(crit_);
+ video_coding::Histogram reordering_histogram_ RTC_GUARDED_BY(crit_);
+ bool initialized_ RTC_GUARDED_BY(crit_);
+ int64_t rtt_ms_ RTC_GUARDED_BY(crit_);
+ uint16_t newest_seq_num_ RTC_GUARDED_BY(crit_);
+
+ // Only touched on the process thread.
+ int64_t next_process_time_ms_;
+
+ // Adds a delay before send nack on packet received.
+ const int64_t send_nack_delay_ms_;
+
+ const absl::optional<BackoffSettings> backoff_settings_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_NACK_MODULE2_H_
diff --git a/modules/video_coding/nack_module2_unittest.cc b/modules/video_coding/nack_module2_unittest.cc
new file mode 100644
index 0000000..ebc28ec
--- /dev/null
+++ b/modules/video_coding/nack_module2_unittest.cc
@@ -0,0 +1,371 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/nack_module2.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <cstring>
+#include <memory>
+
+#include "system_wrappers/include/clock.h"
+#include "test/field_trial.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+class TestNackModule2 : public ::testing::TestWithParam<bool>,
+ public NackSender,
+ public KeyFrameRequestSender {
+ protected:
+ TestNackModule2()
+ : clock_(new SimulatedClock(0)),
+ field_trial_(GetParam()
+ ? "WebRTC-ExponentialNackBackoff/enabled:true/"
+ : "WebRTC-ExponentialNackBackoff/enabled:false/"),
+ nack_module_(clock_.get(), this, this),
+ keyframes_requested_(0) {}
+
+ void SetUp() override { nack_module_.UpdateRtt(kDefaultRttMs); }
+
+ void SendNack(const std::vector<uint16_t>& sequence_numbers,
+ bool buffering_allowed) override {
+ sent_nacks_.insert(sent_nacks_.end(), sequence_numbers.begin(),
+ sequence_numbers.end());
+ }
+
+ void RequestKeyFrame() override { ++keyframes_requested_; }
+
+ static constexpr int64_t kDefaultRttMs = 20;
+ std::unique_ptr<SimulatedClock> clock_;
+ test::ScopedFieldTrials field_trial_;
+ NackModule2 nack_module_;
+ std::vector<uint16_t> sent_nacks_;
+ int keyframes_requested_;
+};
+
+TEST_P(TestNackModule2, NackOnePacket) {
+ nack_module_.OnReceivedPacket(1, false, false);
+ nack_module_.OnReceivedPacket(3, false, false);
+ EXPECT_EQ(1u, sent_nacks_.size());
+ EXPECT_EQ(2, sent_nacks_[0]);
+}
+
+TEST_P(TestNackModule2, WrappingSeqNum) {
+ nack_module_.OnReceivedPacket(0xfffe, false, false);
+ nack_module_.OnReceivedPacket(1, false, false);
+ EXPECT_EQ(2u, sent_nacks_.size());
+ EXPECT_EQ(0xffff, sent_nacks_[0]);
+ EXPECT_EQ(0, sent_nacks_[1]);
+}
+
+TEST_P(TestNackModule2, WrappingSeqNumClearToKeyframe) {
+ nack_module_.OnReceivedPacket(0xfffe, false, false);
+ nack_module_.OnReceivedPacket(1, false, false);
+ EXPECT_EQ(2u, sent_nacks_.size());
+ EXPECT_EQ(0xffff, sent_nacks_[0]);
+ EXPECT_EQ(0, sent_nacks_[1]);
+
+ sent_nacks_.clear();
+ nack_module_.OnReceivedPacket(2, true, false);
+ EXPECT_EQ(0u, sent_nacks_.size());
+
+ nack_module_.OnReceivedPacket(501, true, false);
+ EXPECT_EQ(498u, sent_nacks_.size());
+ for (int seq_num = 3; seq_num < 501; ++seq_num)
+ EXPECT_EQ(seq_num, sent_nacks_[seq_num - 3]);
+
+ sent_nacks_.clear();
+ nack_module_.OnReceivedPacket(1001, false, false);
+ EXPECT_EQ(499u, sent_nacks_.size());
+ for (int seq_num = 502; seq_num < 1001; ++seq_num)
+ EXPECT_EQ(seq_num, sent_nacks_[seq_num - 502]);
+
+ sent_nacks_.clear();
+ clock_->AdvanceTimeMilliseconds(100);
+ nack_module_.Process();
+ EXPECT_EQ(999u, sent_nacks_.size());
+ EXPECT_EQ(0xffff, sent_nacks_[0]);
+ EXPECT_EQ(0, sent_nacks_[1]);
+ for (int seq_num = 3; seq_num < 501; ++seq_num)
+ EXPECT_EQ(seq_num, sent_nacks_[seq_num - 1]);
+ for (int seq_num = 502; seq_num < 1001; ++seq_num)
+ EXPECT_EQ(seq_num, sent_nacks_[seq_num - 2]);
+
+ // Adding packet 1004 will cause the nack list to reach it's max limit.
+ // It will then clear all nacks up to the next keyframe (seq num 2),
+ // thus removing 0xffff and 0 from the nack list.
+ sent_nacks_.clear();
+ nack_module_.OnReceivedPacket(1004, false, false);
+ EXPECT_EQ(2u, sent_nacks_.size());
+ EXPECT_EQ(1002, sent_nacks_[0]);
+ EXPECT_EQ(1003, sent_nacks_[1]);
+
+ sent_nacks_.clear();
+ clock_->AdvanceTimeMilliseconds(100);
+ nack_module_.Process();
+ EXPECT_EQ(999u, sent_nacks_.size());
+ for (int seq_num = 3; seq_num < 501; ++seq_num)
+ EXPECT_EQ(seq_num, sent_nacks_[seq_num - 3]);
+ for (int seq_num = 502; seq_num < 1001; ++seq_num)
+ EXPECT_EQ(seq_num, sent_nacks_[seq_num - 4]);
+
+ // Adding packet 1007 will cause the nack module to overflow again, thus
+ // clearing everything up to 501 which is the next keyframe.
+ nack_module_.OnReceivedPacket(1007, false, false);
+ sent_nacks_.clear();
+ clock_->AdvanceTimeMilliseconds(100);
+ nack_module_.Process();
+ EXPECT_EQ(503u, sent_nacks_.size());
+ for (int seq_num = 502; seq_num < 1001; ++seq_num)
+ EXPECT_EQ(seq_num, sent_nacks_[seq_num - 502]);
+ EXPECT_EQ(1005, sent_nacks_[501]);
+ EXPECT_EQ(1006, sent_nacks_[502]);
+}
+
+TEST_P(TestNackModule2, DontBurstOnTimeSkip) {
+ nack_module_.Process();
+ clock_->AdvanceTimeMilliseconds(20);
+ EXPECT_EQ(0, nack_module_.TimeUntilNextProcess());
+ nack_module_.Process();
+
+ clock_->AdvanceTimeMilliseconds(100);
+ EXPECT_EQ(0, nack_module_.TimeUntilNextProcess());
+ nack_module_.Process();
+ EXPECT_EQ(20, nack_module_.TimeUntilNextProcess());
+
+ clock_->AdvanceTimeMilliseconds(19);
+ EXPECT_EQ(1, nack_module_.TimeUntilNextProcess());
+ clock_->AdvanceTimeMilliseconds(2);
+ nack_module_.Process();
+ EXPECT_EQ(19, nack_module_.TimeUntilNextProcess());
+
+ clock_->AdvanceTimeMilliseconds(19);
+ EXPECT_EQ(0, nack_module_.TimeUntilNextProcess());
+ nack_module_.Process();
+
+ clock_->AdvanceTimeMilliseconds(21);
+ EXPECT_EQ(0, nack_module_.TimeUntilNextProcess());
+ nack_module_.Process();
+ EXPECT_EQ(19, nack_module_.TimeUntilNextProcess());
+}
+
+TEST_P(TestNackModule2, ResendNack) {
+ nack_module_.OnReceivedPacket(1, false, false);
+ nack_module_.OnReceivedPacket(3, false, false);
+ size_t expected_nacks_sent = 1;
+ EXPECT_EQ(expected_nacks_sent, sent_nacks_.size());
+ EXPECT_EQ(2, sent_nacks_[0]);
+
+ if (GetParam()) {
+ // Retry has to wait at least 5ms by default.
+ nack_module_.UpdateRtt(1);
+ clock_->AdvanceTimeMilliseconds(4);
+ nack_module_.Process(); // Too early.
+ EXPECT_EQ(expected_nacks_sent, sent_nacks_.size());
+
+ clock_->AdvanceTimeMilliseconds(1);
+ nack_module_.Process(); // Now allowed.
+ EXPECT_EQ(++expected_nacks_sent, sent_nacks_.size());
+ } else {
+ nack_module_.UpdateRtt(1);
+ clock_->AdvanceTimeMilliseconds(1);
+ nack_module_.Process(); // Fast retransmit allowed.
+ EXPECT_EQ(++expected_nacks_sent, sent_nacks_.size());
+ }
+
+ // N:th try has to wait b^(N-1) * rtt by default.
+ const double b = GetParam() ? 1.25 : 1.0;
+ for (int i = 2; i < 10; ++i) {
+ // Change RTT, above the 40ms max for exponential backoff.
+ TimeDelta rtt = TimeDelta::Millis(160); // + (i * 10 - 40)
+ nack_module_.UpdateRtt(rtt.ms());
+
+ // RTT gets capped at 160ms in backoff calculations.
+ TimeDelta expected_backoff_delay =
+ std::pow(b, i - 1) * std::min(rtt, TimeDelta::Millis(160));
+
+ // Move to one millisecond before next allowed NACK.
+ clock_->AdvanceTimeMilliseconds(expected_backoff_delay.ms() - 1);
+ nack_module_.Process();
+ EXPECT_EQ(expected_nacks_sent, sent_nacks_.size());
+
+ // Move to one millisecond after next allowed NACK.
+ // After rather than on to avoid rounding errors.
+ clock_->AdvanceTimeMilliseconds(2);
+ nack_module_.Process(); // Now allowed.
+ EXPECT_EQ(++expected_nacks_sent, sent_nacks_.size());
+ }
+
+ // Giving up after 10 tries.
+ clock_->AdvanceTimeMilliseconds(3000);
+ nack_module_.Process();
+ EXPECT_EQ(expected_nacks_sent, sent_nacks_.size());
+}
+
+TEST_P(TestNackModule2, ResendPacketMaxRetries) {
+ nack_module_.OnReceivedPacket(1, false, false);
+ nack_module_.OnReceivedPacket(3, false, false);
+ EXPECT_EQ(1u, sent_nacks_.size());
+ EXPECT_EQ(2, sent_nacks_[0]);
+
+ int backoff_factor = 1;
+ for (size_t retries = 1; retries < 10; ++retries) {
+ // Exponential backoff, so that we don't reject NACK because of time.
+ clock_->AdvanceTimeMilliseconds(backoff_factor * kDefaultRttMs);
+ backoff_factor *= 2;
+ nack_module_.Process();
+ EXPECT_EQ(retries + 1, sent_nacks_.size());
+ }
+
+ clock_->AdvanceTimeMilliseconds(backoff_factor * kDefaultRttMs);
+ nack_module_.Process();
+ EXPECT_EQ(10u, sent_nacks_.size());
+}
+
+TEST_P(TestNackModule2, TooLargeNackList) {
+ nack_module_.OnReceivedPacket(0, false, false);
+ nack_module_.OnReceivedPacket(1001, false, false);
+ EXPECT_EQ(1000u, sent_nacks_.size());
+ EXPECT_EQ(0, keyframes_requested_);
+ nack_module_.OnReceivedPacket(1003, false, false);
+ EXPECT_EQ(1000u, sent_nacks_.size());
+ EXPECT_EQ(1, keyframes_requested_);
+ nack_module_.OnReceivedPacket(1004, false, false);
+ EXPECT_EQ(1000u, sent_nacks_.size());
+ EXPECT_EQ(1, keyframes_requested_);
+}
+
+TEST_P(TestNackModule2, TooLargeNackListWithKeyFrame) {
+ nack_module_.OnReceivedPacket(0, false, false);
+ nack_module_.OnReceivedPacket(1, true, false);
+ nack_module_.OnReceivedPacket(1001, false, false);
+ EXPECT_EQ(999u, sent_nacks_.size());
+ EXPECT_EQ(0, keyframes_requested_);
+ nack_module_.OnReceivedPacket(1003, false, false);
+ EXPECT_EQ(1000u, sent_nacks_.size());
+ EXPECT_EQ(0, keyframes_requested_);
+ nack_module_.OnReceivedPacket(1005, false, false);
+ EXPECT_EQ(1000u, sent_nacks_.size());
+ EXPECT_EQ(1, keyframes_requested_);
+}
+
+TEST_P(TestNackModule2, ClearUpTo) {
+ nack_module_.OnReceivedPacket(0, false, false);
+ nack_module_.OnReceivedPacket(100, false, false);
+ EXPECT_EQ(99u, sent_nacks_.size());
+
+ sent_nacks_.clear();
+ clock_->AdvanceTimeMilliseconds(100);
+ nack_module_.ClearUpTo(50);
+ nack_module_.Process();
+ EXPECT_EQ(50u, sent_nacks_.size());
+ EXPECT_EQ(50, sent_nacks_[0]);
+}
+
+TEST_P(TestNackModule2, ClearUpToWrap) {
+ nack_module_.OnReceivedPacket(0xfff0, false, false);
+ nack_module_.OnReceivedPacket(0xf, false, false);
+ EXPECT_EQ(30u, sent_nacks_.size());
+
+ sent_nacks_.clear();
+ clock_->AdvanceTimeMilliseconds(100);
+ nack_module_.ClearUpTo(0);
+ nack_module_.Process();
+ EXPECT_EQ(15u, sent_nacks_.size());
+ EXPECT_EQ(0, sent_nacks_[0]);
+}
+
+TEST_P(TestNackModule2, PacketNackCount) {
+ EXPECT_EQ(0, nack_module_.OnReceivedPacket(0, false, false));
+ EXPECT_EQ(0, nack_module_.OnReceivedPacket(2, false, false));
+ EXPECT_EQ(1, nack_module_.OnReceivedPacket(1, false, false));
+
+ sent_nacks_.clear();
+ nack_module_.UpdateRtt(100);
+ EXPECT_EQ(0, nack_module_.OnReceivedPacket(5, false, false));
+ clock_->AdvanceTimeMilliseconds(100);
+ nack_module_.Process();
+ clock_->AdvanceTimeMilliseconds(125);
+ nack_module_.Process();
+ EXPECT_EQ(3, nack_module_.OnReceivedPacket(3, false, false));
+ EXPECT_EQ(3, nack_module_.OnReceivedPacket(4, false, false));
+ EXPECT_EQ(0, nack_module_.OnReceivedPacket(4, false, false));
+}
+
+TEST_P(TestNackModule2, NackListFullAndNoOverlapWithKeyframes) {
+ const int kMaxNackPackets = 1000;
+ const unsigned int kFirstGap = kMaxNackPackets - 20;
+ const unsigned int kSecondGap = 200;
+ uint16_t seq_num = 0;
+ nack_module_.OnReceivedPacket(seq_num++, true, false);
+ seq_num += kFirstGap;
+ nack_module_.OnReceivedPacket(seq_num++, true, false);
+ EXPECT_EQ(kFirstGap, sent_nacks_.size());
+ sent_nacks_.clear();
+ seq_num += kSecondGap;
+ nack_module_.OnReceivedPacket(seq_num, true, false);
+ EXPECT_EQ(kSecondGap, sent_nacks_.size());
+}
+
+TEST_P(TestNackModule2, HandleFecRecoveredPacket) {
+ nack_module_.OnReceivedPacket(1, false, false);
+ nack_module_.OnReceivedPacket(4, false, true);
+ EXPECT_EQ(0u, sent_nacks_.size());
+ nack_module_.OnReceivedPacket(5, false, false);
+ EXPECT_EQ(2u, sent_nacks_.size());
+}
+
+TEST_P(TestNackModule2, SendNackWithoutDelay) {
+ nack_module_.OnReceivedPacket(0, false, false);
+ nack_module_.OnReceivedPacket(100, false, false);
+ EXPECT_EQ(99u, sent_nacks_.size());
+}
+
+INSTANTIATE_TEST_SUITE_P(WithAndWithoutBackoff,
+ TestNackModule2,
+ ::testing::Values(true, false));
+
+class TestNackModule2WithFieldTrial : public ::testing::Test,
+ public NackSender,
+ public KeyFrameRequestSender {
+ protected:
+ TestNackModule2WithFieldTrial()
+ : nack_delay_field_trial_("WebRTC-SendNackDelayMs/10/"),
+ clock_(new SimulatedClock(0)),
+ nack_module_(clock_.get(), this, this),
+ keyframes_requested_(0) {}
+
+ void SendNack(const std::vector<uint16_t>& sequence_numbers,
+ bool buffering_allowed) override {
+ sent_nacks_.insert(sent_nacks_.end(), sequence_numbers.begin(),
+ sequence_numbers.end());
+ }
+
+ void RequestKeyFrame() override { ++keyframes_requested_; }
+
+ test::ScopedFieldTrials nack_delay_field_trial_;
+ std::unique_ptr<SimulatedClock> clock_;
+ NackModule2 nack_module_;
+ std::vector<uint16_t> sent_nacks_;
+ int keyframes_requested_;
+};
+
+TEST_F(TestNackModule2WithFieldTrial, SendNackWithDelay) {
+ nack_module_.OnReceivedPacket(0, false, false);
+ nack_module_.OnReceivedPacket(100, false, false);
+ EXPECT_EQ(0u, sent_nacks_.size());
+ clock_->AdvanceTimeMilliseconds(10);
+ nack_module_.OnReceivedPacket(106, false, false);
+ EXPECT_EQ(99u, sent_nacks_.size());
+ clock_->AdvanceTimeMilliseconds(10);
+ nack_module_.OnReceivedPacket(109, false, false);
+ EXPECT_EQ(104u, sent_nacks_.size());
+}
+} // namespace webrtc
diff --git a/video/BUILD.gn b/video/BUILD.gn
index 25df508..07b03ee 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -34,6 +34,8 @@
"rtp_streams_synchronizer2.h",
"rtp_video_stream_receiver.cc",
"rtp_video_stream_receiver.h",
+ "rtp_video_stream_receiver2.cc",
+ "rtp_video_stream_receiver2.h",
"rtp_video_stream_receiver_frame_transformer_delegate.cc",
"rtp_video_stream_receiver_frame_transformer_delegate.h",
"send_delay_stats.cc",
@@ -528,6 +530,7 @@
"receive_statistics_proxy2_unittest.cc",
"receive_statistics_proxy_unittest.cc",
"report_block_stats_unittest.cc",
+ "rtp_video_stream_receiver2_unittest.cc",
"rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc",
"rtp_video_stream_receiver_unittest.cc",
"send_delay_stats_unittest.cc",
diff --git a/video/rtp_video_stream_receiver.cc b/video/rtp_video_stream_receiver.cc
index 87691d4..53fd637 100644
--- a/video/rtp_video_stream_receiver.cc
+++ b/video/rtp_video_stream_receiver.cc
@@ -77,8 +77,6 @@
return packet_buffer_max_size;
}
-} // namespace
-
std::unique_ptr<RtpRtcp> CreateRtpRtcpModule(
Clock* clock,
ReceiveStatistics* receive_statistics,
@@ -107,6 +105,8 @@
static const int kPacketLogIntervalMs = 10000;
+} // namespace
+
RtpVideoStreamReceiver::RtcpFeedbackBuffer::RtcpFeedbackBuffer(
KeyFrameRequestSender* key_frame_request_sender,
NackSender* nack_sender,
diff --git a/video/rtp_video_stream_receiver.h b/video/rtp_video_stream_receiver.h
index 32d675c..902c118 100644
--- a/video/rtp_video_stream_receiver.h
+++ b/video/rtp_video_stream_receiver.h
@@ -69,7 +69,8 @@
public KeyFrameRequestSender,
public video_coding::OnCompleteFrameCallback,
public OnDecryptedFrameCallback,
- public OnDecryptionStatusChangeCallback {
+ public OnDecryptionStatusChangeCallback,
+ public RtpVideoFrameReceiver {
public:
// DEPRECATED due to dependency on ReceiveStatisticsProxy.
RtpVideoStreamReceiver(
@@ -205,9 +206,11 @@
void AddSecondarySink(RtpPacketSinkInterface* sink);
void RemoveSecondarySink(const RtpPacketSinkInterface* sink);
- virtual void ManageFrame(std::unique_ptr<video_coding::RtpFrameObject> frame);
-
private:
+ // Implements RtpVideoFrameReceiver.
+ void ManageFrame(
+ std::unique_ptr<video_coding::RtpFrameObject> frame) override;
+
// Used for buffering RTCP feedback messages and sending them all together.
// Note:
// 1. Key frame requests and NACKs are mutually exclusive, with the
diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc
new file mode 100644
index 0000000..98351c4
--- /dev/null
+++ b/video/rtp_video_stream_receiver2.cc
@@ -0,0 +1,1152 @@
+/*
+ * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/rtp_video_stream_receiver2.h"
+
+#include <algorithm>
+#include <limits>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "absl/algorithm/container.h"
+#include "absl/base/macros.h"
+#include "absl/memory/memory.h"
+#include "absl/types/optional.h"
+#include "media/base/media_constants.h"
+#include "modules/pacing/packet_router.h"
+#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
+#include "modules/rtp_rtcp/include/receive_statistics.h"
+#include "modules/rtp_rtcp/include/rtp_cvo.h"
+#include "modules/rtp_rtcp/include/rtp_rtcp.h"
+#include "modules/rtp_rtcp/include/ulpfec_receiver.h"
+#include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h"
+#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
+#include "modules/rtp_rtcp/source/rtp_format.h"
+#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h"
+#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
+#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
+#include "modules/rtp_rtcp/source/rtp_packet_received.h"
+#include "modules/rtp_rtcp/source/rtp_rtcp_config.h"
+#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h"
+#include "modules/rtp_rtcp/source/video_rtp_depacketizer_raw.h"
+#include "modules/utility/include/process_thread.h"
+#include "modules/video_coding/frame_object.h"
+#include "modules/video_coding/h264_sprop_parameter_sets.h"
+#include "modules/video_coding/h264_sps_pps_tracker.h"
+#include "modules/video_coding/nack_module2.h"
+#include "modules/video_coding/packet_buffer.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/location.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/strings/string_builder.h"
+#include "system_wrappers/include/field_trial.h"
+#include "system_wrappers/include/metrics.h"
+#include "system_wrappers/include/ntp_time.h"
+#include "video/receive_statistics_proxy2.h"
+
+namespace webrtc {
+
+namespace {
+// TODO(philipel): Change kPacketBufferStartSize back to 32 in M63 see:
+// crbug.com/752886
+constexpr int kPacketBufferStartSize = 512;
+constexpr int kPacketBufferMaxSize = 2048;
+
+int PacketBufferMaxSize() {
+ // The group here must be a positive power of 2, in which case that is used as
+ // size. All other values shall result in the default value being used.
+ const std::string group_name =
+ webrtc::field_trial::FindFullName("WebRTC-PacketBufferMaxSize");
+ int packet_buffer_max_size = kPacketBufferMaxSize;
+ if (!group_name.empty() &&
+ (sscanf(group_name.c_str(), "%d", &packet_buffer_max_size) != 1 ||
+ packet_buffer_max_size <= 0 ||
+ // Verify that the number is a positive power of 2.
+ (packet_buffer_max_size & (packet_buffer_max_size - 1)) != 0)) {
+ RTC_LOG(LS_WARNING) << "Invalid packet buffer max size: " << group_name;
+ packet_buffer_max_size = kPacketBufferMaxSize;
+ }
+ return packet_buffer_max_size;
+}
+
+std::unique_ptr<RtpRtcp> CreateRtpRtcpModule(
+ Clock* clock,
+ ReceiveStatistics* receive_statistics,
+ Transport* outgoing_transport,
+ RtcpRttStats* rtt_stats,
+ RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
+ RtcpCnameCallback* rtcp_cname_callback,
+ uint32_t local_ssrc) {
+ RtpRtcp::Configuration configuration;
+ configuration.clock = clock;
+ configuration.audio = false;
+ configuration.receiver_only = true;
+ configuration.receive_statistics = receive_statistics;
+ configuration.outgoing_transport = outgoing_transport;
+ configuration.rtt_stats = rtt_stats;
+ configuration.rtcp_packet_type_counter_observer =
+ rtcp_packet_type_counter_observer;
+ configuration.rtcp_cname_callback = rtcp_cname_callback;
+ configuration.local_media_ssrc = local_ssrc;
+
+ std::unique_ptr<RtpRtcp> rtp_rtcp = RtpRtcp::Create(configuration);
+ rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound);
+
+ return rtp_rtcp;
+}
+
+static const int kPacketLogIntervalMs = 10000;
+} // namespace
+
+RtpVideoStreamReceiver2::RtcpFeedbackBuffer::RtcpFeedbackBuffer(
+ KeyFrameRequestSender* key_frame_request_sender,
+ NackSender* nack_sender,
+ LossNotificationSender* loss_notification_sender)
+ : key_frame_request_sender_(key_frame_request_sender),
+ nack_sender_(nack_sender),
+ loss_notification_sender_(loss_notification_sender),
+ request_key_frame_(false) {
+ RTC_DCHECK(key_frame_request_sender_);
+ RTC_DCHECK(nack_sender_);
+ RTC_DCHECK(loss_notification_sender_);
+}
+
+void RtpVideoStreamReceiver2::RtcpFeedbackBuffer::RequestKeyFrame() {
+ rtc::CritScope lock(&cs_);
+ request_key_frame_ = true;
+}
+
+void RtpVideoStreamReceiver2::RtcpFeedbackBuffer::SendNack(
+ const std::vector<uint16_t>& sequence_numbers,
+ bool buffering_allowed) {
+ RTC_DCHECK(!sequence_numbers.empty());
+ rtc::CritScope lock(&cs_);
+ nack_sequence_numbers_.insert(nack_sequence_numbers_.end(),
+ sequence_numbers.cbegin(),
+ sequence_numbers.cend());
+ if (!buffering_allowed) {
+ // Note that while *buffering* is not allowed, *batching* is, meaning that
+ // previously buffered messages may be sent along with the current message.
+ SendRtcpFeedback(ConsumeRtcpFeedbackLocked());
+ }
+}
+
+void RtpVideoStreamReceiver2::RtcpFeedbackBuffer::SendLossNotification(
+ uint16_t last_decoded_seq_num,
+ uint16_t last_received_seq_num,
+ bool decodability_flag,
+ bool buffering_allowed) {
+ RTC_DCHECK(buffering_allowed);
+ rtc::CritScope lock(&cs_);
+ RTC_DCHECK(!lntf_state_)
+ << "SendLossNotification() called twice in a row with no call to "
+ "SendBufferedRtcpFeedback() in between.";
+ lntf_state_ = absl::make_optional<LossNotificationState>(
+ last_decoded_seq_num, last_received_seq_num, decodability_flag);
+}
+
+void RtpVideoStreamReceiver2::RtcpFeedbackBuffer::SendBufferedRtcpFeedback() {
+ SendRtcpFeedback(ConsumeRtcpFeedback());
+}
+
+RtpVideoStreamReceiver2::RtcpFeedbackBuffer::ConsumedRtcpFeedback
+RtpVideoStreamReceiver2::RtcpFeedbackBuffer::ConsumeRtcpFeedback() {
+ rtc::CritScope lock(&cs_);
+ return ConsumeRtcpFeedbackLocked();
+}
+
+RtpVideoStreamReceiver2::RtcpFeedbackBuffer::ConsumedRtcpFeedback
+RtpVideoStreamReceiver2::RtcpFeedbackBuffer::ConsumeRtcpFeedbackLocked() {
+ ConsumedRtcpFeedback feedback;
+ std::swap(feedback.request_key_frame, request_key_frame_);
+ std::swap(feedback.nack_sequence_numbers, nack_sequence_numbers_);
+ std::swap(feedback.lntf_state, lntf_state_);
+ return feedback;
+}
+
+void RtpVideoStreamReceiver2::RtcpFeedbackBuffer::SendRtcpFeedback(
+ ConsumedRtcpFeedback feedback) {
+ if (feedback.lntf_state) {
+ // If either a NACK or a key frame request is sent, we should buffer
+ // the LNTF and wait for them (NACK or key frame request) to trigger
+ // the compound feedback message.
+ // Otherwise, the LNTF should be sent out immediately.
+ const bool buffering_allowed =
+ feedback.request_key_frame || !feedback.nack_sequence_numbers.empty();
+
+ loss_notification_sender_->SendLossNotification(
+ feedback.lntf_state->last_decoded_seq_num,
+ feedback.lntf_state->last_received_seq_num,
+ feedback.lntf_state->decodability_flag, buffering_allowed);
+ }
+
+ if (feedback.request_key_frame) {
+ key_frame_request_sender_->RequestKeyFrame();
+ } else if (!feedback.nack_sequence_numbers.empty()) {
+ nack_sender_->SendNack(feedback.nack_sequence_numbers, true);
+ }
+}
+
+RtpVideoStreamReceiver2::RtpVideoStreamReceiver2(
+ Clock* clock,
+ Transport* transport,
+ RtcpRttStats* rtt_stats,
+ PacketRouter* packet_router,
+ const VideoReceiveStream::Config* config,
+ ReceiveStatistics* rtp_receive_statistics,
+ RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
+ RtcpCnameCallback* rtcp_cname_callback,
+ ProcessThread* process_thread,
+ NackSender* nack_sender,
+ KeyFrameRequestSender* keyframe_request_sender,
+ video_coding::OnCompleteFrameCallback* complete_frame_callback,
+ rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
+ rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)
+ : clock_(clock),
+ config_(*config),
+ packet_router_(packet_router),
+ process_thread_(process_thread),
+ ntp_estimator_(clock),
+ rtp_header_extensions_(config_.rtp.extensions),
+ forced_playout_delay_max_ms_("max_ms", absl::nullopt),
+ forced_playout_delay_min_ms_("min_ms", absl::nullopt),
+ rtp_receive_statistics_(rtp_receive_statistics),
+ ulpfec_receiver_(UlpfecReceiver::Create(config->rtp.remote_ssrc,
+ this,
+ config->rtp.extensions)),
+ receiving_(false),
+ last_packet_log_ms_(-1),
+ rtp_rtcp_(CreateRtpRtcpModule(clock,
+ rtp_receive_statistics_,
+ transport,
+ rtt_stats,
+ rtcp_packet_type_counter_observer,
+ rtcp_cname_callback,
+ config_.rtp.local_ssrc)),
+ complete_frame_callback_(complete_frame_callback),
+ keyframe_request_sender_(keyframe_request_sender),
+ // TODO(bugs.webrtc.org/10336): Let |rtcp_feedback_buffer_| communicate
+ // directly with |rtp_rtcp_|.
+ rtcp_feedback_buffer_(this, nack_sender, this),
+ packet_buffer_(clock_, kPacketBufferStartSize, PacketBufferMaxSize()),
+ has_received_frame_(false),
+ frames_decryptable_(false),
+ absolute_capture_time_receiver_(clock) {
+ constexpr bool remb_candidate = true;
+ if (packet_router_)
+ packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate);
+
+ RTC_DCHECK(config_.rtp.rtcp_mode != RtcpMode::kOff)
+ << "A stream should not be configured with RTCP disabled. This value is "
+ "reserved for internal usage.";
+ // TODO(pbos): What's an appropriate local_ssrc for receive-only streams?
+ RTC_DCHECK(config_.rtp.local_ssrc != 0);
+ RTC_DCHECK(config_.rtp.remote_ssrc != config_.rtp.local_ssrc);
+
+ rtp_rtcp_->SetRTCPStatus(config_.rtp.rtcp_mode);
+ rtp_rtcp_->SetRemoteSSRC(config_.rtp.remote_ssrc);
+
+ static const int kMaxPacketAgeToNack = 450;
+ const int max_reordering_threshold = (config_.rtp.nack.rtp_history_ms > 0)
+ ? kMaxPacketAgeToNack
+ : kDefaultMaxReorderingThreshold;
+ rtp_receive_statistics_->SetMaxReorderingThreshold(config_.rtp.remote_ssrc,
+ max_reordering_threshold);
+ // TODO(nisse): For historic reasons, we applied the above
+ // max_reordering_threshold also for RTX stats, which makes little sense since
+ // we don't NACK rtx packets. Consider deleting the below block, and rely on
+ // the default threshold.
+ if (config_.rtp.rtx_ssrc) {
+ rtp_receive_statistics_->SetMaxReorderingThreshold(
+ config_.rtp.rtx_ssrc, max_reordering_threshold);
+ }
+ if (config_.rtp.rtcp_xr.receiver_reference_time_report)
+ rtp_rtcp_->SetRtcpXrRrtrStatus(true);
+
+ ParseFieldTrial(
+ {&forced_playout_delay_max_ms_, &forced_playout_delay_min_ms_},
+ field_trial::FindFullName("WebRTC-ForcePlayoutDelay"));
+
+ process_thread_->RegisterModule(rtp_rtcp_.get(), RTC_FROM_HERE);
+
+ if (config_.rtp.lntf.enabled) {
+ loss_notification_controller_ =
+ std::make_unique<LossNotificationController>(&rtcp_feedback_buffer_,
+ &rtcp_feedback_buffer_);
+ }
+
+ if (config_.rtp.nack.rtp_history_ms != 0) {
+ nack_module_ = std::make_unique<NackModule2>(clock_, &rtcp_feedback_buffer_,
+ &rtcp_feedback_buffer_);
+ process_thread_->RegisterModule(nack_module_.get(), RTC_FROM_HERE);
+ }
+
+ reference_finder_ =
+ std::make_unique<video_coding::RtpFrameReferenceFinder>(this);
+
+ // Only construct the encrypted receiver if frame encryption is enabled.
+ if (config_.crypto_options.sframe.require_frame_encryption) {
+ buffered_frame_decryptor_ =
+ std::make_unique<BufferedFrameDecryptor>(this, this);
+ if (frame_decryptor != nullptr) {
+ buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor));
+ }
+ }
+
+ if (frame_transformer) {
+ frame_transformer_delegate_ = new rtc::RefCountedObject<
+ RtpVideoStreamReceiverFrameTransformerDelegate>(
+ this, std::move(frame_transformer), rtc::Thread::Current(),
+ config_.rtp.remote_ssrc);
+ frame_transformer_delegate_->Init();
+ }
+}
+
+RtpVideoStreamReceiver2::~RtpVideoStreamReceiver2() {
+ RTC_DCHECK(secondary_sinks_.empty());
+
+ if (nack_module_) {
+ process_thread_->DeRegisterModule(nack_module_.get());
+ }
+
+ process_thread_->DeRegisterModule(rtp_rtcp_.get());
+
+ if (packet_router_)
+ packet_router_->RemoveReceiveRtpModule(rtp_rtcp_.get());
+ UpdateHistograms();
+ if (frame_transformer_delegate_)
+ frame_transformer_delegate_->Reset();
+}
+
+void RtpVideoStreamReceiver2::AddReceiveCodec(
+ const VideoCodec& video_codec,
+ const std::map<std::string, std::string>& codec_params,
+ bool raw_payload) {
+ payload_type_map_.emplace(
+ video_codec.plType,
+ raw_payload ? std::make_unique<VideoRtpDepacketizerRaw>()
+ : CreateVideoRtpDepacketizer(video_codec.codecType));
+ pt_codec_params_.emplace(video_codec.plType, codec_params);
+}
+
+absl::optional<Syncable::Info> RtpVideoStreamReceiver2::GetSyncInfo() const {
+ Syncable::Info info;
+ if (rtp_rtcp_->RemoteNTP(&info.capture_time_ntp_secs,
+ &info.capture_time_ntp_frac, nullptr, nullptr,
+ &info.capture_time_source_clock) != 0) {
+ return absl::nullopt;
+ }
+ {
+ rtc::CritScope lock(&sync_info_lock_);
+ if (!last_received_rtp_timestamp_ || !last_received_rtp_system_time_ms_) {
+ return absl::nullopt;
+ }
+ info.latest_received_capture_timestamp = *last_received_rtp_timestamp_;
+ info.latest_receive_time_ms = *last_received_rtp_system_time_ms_;
+ }
+
+ // Leaves info.current_delay_ms uninitialized.
+ return info;
+}
+
+RtpVideoStreamReceiver2::ParseGenericDependenciesResult
+RtpVideoStreamReceiver2::ParseGenericDependenciesExtension(
+ const RtpPacketReceived& rtp_packet,
+ RTPVideoHeader* video_header) {
+ if (rtp_packet.HasExtension<RtpDependencyDescriptorExtension>()) {
+ webrtc::DependencyDescriptor dependency_descriptor;
+ if (!rtp_packet.GetExtension<RtpDependencyDescriptorExtension>(
+ video_structure_.get(), &dependency_descriptor)) {
+ // Descriptor is there, but failed to parse. Either it is invalid,
+ // or too old packet (after relevant video_structure_ changed),
+ // or too new packet (before relevant video_structure_ arrived).
+ // Drop such packet to be on the safe side.
+ // TODO(bugs.webrtc.org/10342): Stash too new packet.
+ RTC_LOG(LS_WARNING) << "ssrc: " << rtp_packet.Ssrc()
+ << " Failed to parse dependency descriptor.";
+ return kDropPacket;
+ }
+ if (dependency_descriptor.attached_structure != nullptr &&
+ !dependency_descriptor.first_packet_in_frame) {
+ RTC_LOG(LS_WARNING) << "ssrc: " << rtp_packet.Ssrc()
+ << "Invalid dependency descriptor: structure "
+ "attached to non first packet of a frame.";
+ return kDropPacket;
+ }
+ video_header->is_first_packet_in_frame =
+ dependency_descriptor.first_packet_in_frame;
+ video_header->is_last_packet_in_frame =
+ dependency_descriptor.last_packet_in_frame;
+
+ int64_t frame_id =
+ frame_id_unwrapper_.Unwrap(dependency_descriptor.frame_number);
+ auto& generic_descriptor_info = video_header->generic.emplace();
+ generic_descriptor_info.frame_id = frame_id;
+ generic_descriptor_info.spatial_index =
+ dependency_descriptor.frame_dependencies.spatial_id;
+ generic_descriptor_info.temporal_index =
+ dependency_descriptor.frame_dependencies.temporal_id;
+ for (int fdiff : dependency_descriptor.frame_dependencies.frame_diffs) {
+ generic_descriptor_info.dependencies.push_back(frame_id - fdiff);
+ }
+ generic_descriptor_info.decode_target_indications =
+ dependency_descriptor.frame_dependencies.decode_target_indications;
+ if (dependency_descriptor.resolution) {
+ video_header->width = dependency_descriptor.resolution->Width();
+ video_header->height = dependency_descriptor.resolution->Height();
+ }
+
+ // FrameDependencyStructure is sent in dependency descriptor of the first
+ // packet of a key frame and required for parsed dependency descriptor in
+ // all the following packets until next key frame.
+ // Save it if there is a (potentially) new structure.
+ if (dependency_descriptor.attached_structure) {
+ RTC_DCHECK(dependency_descriptor.first_packet_in_frame);
+ if (video_structure_frame_id_ > frame_id) {
+ RTC_LOG(LS_WARNING)
+ << "Arrived key frame with id " << frame_id << " and structure id "
+ << dependency_descriptor.attached_structure->structure_id
+ << " is older than the latest received key frame with id "
+ << *video_structure_frame_id_ << " and structure id "
+ << video_structure_->structure_id;
+ return kDropPacket;
+ }
+ video_structure_ = std::move(dependency_descriptor.attached_structure);
+ video_structure_frame_id_ = frame_id;
+ video_header->frame_type = VideoFrameType::kVideoFrameKey;
+ } else {
+ video_header->frame_type = VideoFrameType::kVideoFrameDelta;
+ }
+ return kHasGenericDescriptor;
+ }
+
+ RtpGenericFrameDescriptor generic_frame_descriptor;
+ if (!rtp_packet.GetExtension<RtpGenericFrameDescriptorExtension00>(
+ &generic_frame_descriptor)) {
+ return kNoGenericDescriptor;
+ }
+
+ video_header->is_first_packet_in_frame =
+ generic_frame_descriptor.FirstPacketInSubFrame();
+ video_header->is_last_packet_in_frame =
+ generic_frame_descriptor.LastPacketInSubFrame();
+
+ if (generic_frame_descriptor.FirstPacketInSubFrame()) {
+ video_header->frame_type =
+ generic_frame_descriptor.FrameDependenciesDiffs().empty()
+ ? VideoFrameType::kVideoFrameKey
+ : VideoFrameType::kVideoFrameDelta;
+
+ auto& generic_descriptor_info = video_header->generic.emplace();
+ int64_t frame_id =
+ frame_id_unwrapper_.Unwrap(generic_frame_descriptor.FrameId());
+ generic_descriptor_info.frame_id = frame_id;
+ generic_descriptor_info.spatial_index =
+ generic_frame_descriptor.SpatialLayer();
+ generic_descriptor_info.temporal_index =
+ generic_frame_descriptor.TemporalLayer();
+ for (uint16_t fdiff : generic_frame_descriptor.FrameDependenciesDiffs()) {
+ generic_descriptor_info.dependencies.push_back(frame_id - fdiff);
+ }
+ }
+ video_header->width = generic_frame_descriptor.Width();
+ video_header->height = generic_frame_descriptor.Height();
+ return kHasGenericDescriptor;
+}
+
+void RtpVideoStreamReceiver2::OnReceivedPayloadData(
+ rtc::CopyOnWriteBuffer codec_payload,
+ const RtpPacketReceived& rtp_packet,
+ const RTPVideoHeader& video) {
+ RTC_DCHECK_RUN_ON(&worker_task_checker_);
+ auto packet = std::make_unique<video_coding::PacketBuffer::Packet>(
+ rtp_packet, video, ntp_estimator_.Estimate(rtp_packet.Timestamp()),
+ clock_->TimeInMilliseconds());
+
+ // Try to extrapolate absolute capture time if it is missing.
+ packet->packet_info.set_absolute_capture_time(
+ absolute_capture_time_receiver_.OnReceivePacket(
+ AbsoluteCaptureTimeReceiver::GetSource(packet->packet_info.ssrc(),
+ packet->packet_info.csrcs()),
+ packet->packet_info.rtp_timestamp(),
+ // Assume frequency is the same one for all video frames.
+ kVideoPayloadTypeFrequency,
+ packet->packet_info.absolute_capture_time()));
+
+ RTPVideoHeader& video_header = packet->video_header;
+ video_header.rotation = kVideoRotation_0;
+ video_header.content_type = VideoContentType::UNSPECIFIED;
+ video_header.video_timing.flags = VideoSendTiming::kInvalid;
+ video_header.is_last_packet_in_frame |= rtp_packet.Marker();
+ video_header.frame_marking.temporal_id = kNoTemporalIdx;
+
+ if (const auto* vp9_header =
+ absl::get_if<RTPVideoHeaderVP9>(&video_header.video_type_header)) {
+ video_header.is_last_packet_in_frame |= vp9_header->end_of_frame;
+ video_header.is_first_packet_in_frame |= vp9_header->beginning_of_frame;
+ }
+
+ rtp_packet.GetExtension<VideoOrientation>(&video_header.rotation);
+ rtp_packet.GetExtension<VideoContentTypeExtension>(
+ &video_header.content_type);
+ rtp_packet.GetExtension<VideoTimingExtension>(&video_header.video_timing);
+ if (forced_playout_delay_max_ms_ && forced_playout_delay_min_ms_) {
+ video_header.playout_delay.max_ms = *forced_playout_delay_max_ms_;
+ video_header.playout_delay.min_ms = *forced_playout_delay_min_ms_;
+ } else {
+ rtp_packet.GetExtension<PlayoutDelayLimits>(&video_header.playout_delay);
+ }
+ rtp_packet.GetExtension<FrameMarkingExtension>(&video_header.frame_marking);
+
+ ParseGenericDependenciesResult generic_descriptor_state =
+ ParseGenericDependenciesExtension(rtp_packet, &video_header);
+ if (generic_descriptor_state == kDropPacket)
+ return;
+
+ // Color space should only be transmitted in the last packet of a frame,
+ // therefore, neglect it otherwise so that last_color_space_ is not reset by
+ // mistake.
+ if (video_header.is_last_packet_in_frame) {
+ video_header.color_space = rtp_packet.GetExtension<ColorSpaceExtension>();
+ if (video_header.color_space ||
+ video_header.frame_type == VideoFrameType::kVideoFrameKey) {
+ // Store color space since it's only transmitted when changed or for key
+ // frames. Color space will be cleared if a key frame is transmitted
+ // without color space information.
+ last_color_space_ = video_header.color_space;
+ } else if (last_color_space_) {
+ video_header.color_space = last_color_space_;
+ }
+ }
+
+ if (loss_notification_controller_) {
+ if (rtp_packet.recovered()) {
+ // TODO(bugs.webrtc.org/10336): Implement support for reordering.
+ RTC_LOG(LS_INFO)
+ << "LossNotificationController does not support reordering.";
+ } else if (generic_descriptor_state == kNoGenericDescriptor) {
+ RTC_LOG(LS_WARNING) << "LossNotificationController requires generic "
+ "frame descriptor, but it is missing.";
+ } else {
+ if (video_header.is_first_packet_in_frame) {
+ RTC_DCHECK(video_header.generic);
+ LossNotificationController::FrameDetails frame;
+ frame.is_keyframe =
+ video_header.frame_type == VideoFrameType::kVideoFrameKey;
+ frame.frame_id = video_header.generic->frame_id;
+ frame.frame_dependencies = video_header.generic->dependencies;
+ loss_notification_controller_->OnReceivedPacket(
+ rtp_packet.SequenceNumber(), &frame);
+ } else {
+ loss_notification_controller_->OnReceivedPacket(
+ rtp_packet.SequenceNumber(), nullptr);
+ }
+ }
+ }
+
+ if (nack_module_) {
+ const bool is_keyframe =
+ video_header.is_first_packet_in_frame &&
+ video_header.frame_type == VideoFrameType::kVideoFrameKey;
+
+ packet->times_nacked = nack_module_->OnReceivedPacket(
+ rtp_packet.SequenceNumber(), is_keyframe, rtp_packet.recovered());
+ } else {
+ packet->times_nacked = -1;
+ }
+
+ if (codec_payload.size() == 0) {
+ NotifyReceiverOfEmptyPacket(packet->seq_num);
+ rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
+ return;
+ }
+
+ if (packet->codec() == kVideoCodecH264) {
+ // Only when we start to receive packets will we know what payload type
+ // that will be used. When we know the payload type insert the correct
+ // sps/pps into the tracker.
+ if (packet->payload_type != last_payload_type_) {
+ last_payload_type_ = packet->payload_type;
+ InsertSpsPpsIntoTracker(packet->payload_type);
+ }
+
+ video_coding::H264SpsPpsTracker::FixedBitstream fixed =
+ tracker_.CopyAndFixBitstream(
+ rtc::MakeArrayView(codec_payload.cdata(), codec_payload.size()),
+ &packet->video_header);
+
+ switch (fixed.action) {
+ case video_coding::H264SpsPpsTracker::kRequestKeyframe:
+ rtcp_feedback_buffer_.RequestKeyFrame();
+ rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
+ ABSL_FALLTHROUGH_INTENDED;
+ case video_coding::H264SpsPpsTracker::kDrop:
+ return;
+ case video_coding::H264SpsPpsTracker::kInsert:
+ packet->video_payload = std::move(fixed.bitstream);
+ break;
+ }
+
+ } else {
+ packet->video_payload = std::move(codec_payload);
+ }
+
+ rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
+ frame_counter_.Add(packet->timestamp);
+ OnInsertedPacket(packet_buffer_.InsertPacket(std::move(packet)));
+}
+
+void RtpVideoStreamReceiver2::OnRecoveredPacket(const uint8_t* rtp_packet,
+ size_t rtp_packet_length) {
+ RtpPacketReceived packet;
+ if (!packet.Parse(rtp_packet, rtp_packet_length))
+ return;
+ if (packet.PayloadType() == config_.rtp.red_payload_type) {
+ RTC_LOG(LS_WARNING) << "Discarding recovered packet with RED encapsulation";
+ return;
+ }
+
+ packet.IdentifyExtensions(rtp_header_extensions_);
+ packet.set_payload_type_frequency(kVideoPayloadTypeFrequency);
+ // TODO(nisse): UlpfecReceiverImpl::ProcessReceivedFec passes both
+ // original (decapsulated) media packets and recovered packets to
+ // this callback. We need a way to distinguish, for setting
+ // packet.recovered() correctly. Ideally, move RED decapsulation out
+ // of the Ulpfec implementation.
+
+ ReceivePacket(packet);
+}
+
+// This method handles both regular RTP packets and packets recovered
+// via FlexFEC.
+void RtpVideoStreamReceiver2::OnRtpPacket(const RtpPacketReceived& packet) {
+ RTC_DCHECK_RUN_ON(&worker_task_checker_);
+
+ if (!receiving_) {
+ return;
+ }
+
+ if (!packet.recovered()) {
+ // TODO(nisse): Exclude out-of-order packets?
+ int64_t now_ms = clock_->TimeInMilliseconds();
+ {
+ rtc::CritScope cs(&sync_info_lock_);
+ last_received_rtp_timestamp_ = packet.Timestamp();
+ last_received_rtp_system_time_ms_ = now_ms;
+ }
+ // Periodically log the RTP header of incoming packets.
+ if (now_ms - last_packet_log_ms_ > kPacketLogIntervalMs) {
+ rtc::StringBuilder ss;
+ ss << "Packet received on SSRC: " << packet.Ssrc()
+ << " with payload type: " << static_cast<int>(packet.PayloadType())
+ << ", timestamp: " << packet.Timestamp()
+ << ", sequence number: " << packet.SequenceNumber()
+ << ", arrival time: " << packet.arrival_time_ms();
+ int32_t time_offset;
+ if (packet.GetExtension<TransmissionOffset>(&time_offset)) {
+ ss << ", toffset: " << time_offset;
+ }
+ uint32_t send_time;
+ if (packet.GetExtension<AbsoluteSendTime>(&send_time)) {
+ ss << ", abs send time: " << send_time;
+ }
+ RTC_LOG(LS_INFO) << ss.str();
+ last_packet_log_ms_ = now_ms;
+ }
+ }
+
+ ReceivePacket(packet);
+
+ // Update receive statistics after ReceivePacket.
+ // Receive statistics will be reset if the payload type changes (make sure
+ // that the first packet is included in the stats).
+ if (!packet.recovered()) {
+ rtp_receive_statistics_->OnRtpPacket(packet);
+ }
+
+ for (RtpPacketSinkInterface* secondary_sink : secondary_sinks_) {
+ secondary_sink->OnRtpPacket(packet);
+ }
+}
+
+void RtpVideoStreamReceiver2::RequestKeyFrame() {
+ // TODO(bugs.webrtc.org/10336): Allow the sender to ignore key frame requests
+ // issued by anything other than the LossNotificationController if it (the
+ // sender) is relying on LNTF alone.
+ if (keyframe_request_sender_) {
+ keyframe_request_sender_->RequestKeyFrame();
+ } else {
+ rtp_rtcp_->SendPictureLossIndication();
+ }
+}
+
+void RtpVideoStreamReceiver2::SendLossNotification(
+ uint16_t last_decoded_seq_num,
+ uint16_t last_received_seq_num,
+ bool decodability_flag,
+ bool buffering_allowed) {
+ RTC_DCHECK(config_.rtp.lntf.enabled);
+ rtp_rtcp_->SendLossNotification(last_decoded_seq_num, last_received_seq_num,
+ decodability_flag, buffering_allowed);
+}
+
+bool RtpVideoStreamReceiver2::IsUlpfecEnabled() const {
+ return config_.rtp.ulpfec_payload_type != -1;
+}
+
+bool RtpVideoStreamReceiver2::IsRetransmissionsEnabled() const {
+ return config_.rtp.nack.rtp_history_ms > 0;
+}
+
+void RtpVideoStreamReceiver2::RequestPacketRetransmit(
+ const std::vector<uint16_t>& sequence_numbers) {
+ rtp_rtcp_->SendNack(sequence_numbers);
+}
+
+bool RtpVideoStreamReceiver2::IsDecryptable() const {
+ return frames_decryptable_.load();
+}
+
+void RtpVideoStreamReceiver2::OnInsertedPacket(
+ video_coding::PacketBuffer::InsertResult result) {
+ video_coding::PacketBuffer::Packet* first_packet = nullptr;
+ int max_nack_count;
+ int64_t min_recv_time;
+ int64_t max_recv_time;
+ std::vector<rtc::ArrayView<const uint8_t>> payloads;
+ RtpPacketInfos::vector_type packet_infos;
+
+ bool frame_boundary = true;
+ for (auto& packet : result.packets) {
+ // PacketBuffer promisses frame boundaries are correctly set on each
+ // packet. Document that assumption with the DCHECKs.
+ RTC_DCHECK_EQ(frame_boundary, packet->is_first_packet_in_frame());
+ if (packet->is_first_packet_in_frame()) {
+ first_packet = packet.get();
+ max_nack_count = packet->times_nacked;
+ min_recv_time = packet->packet_info.receive_time_ms();
+ max_recv_time = packet->packet_info.receive_time_ms();
+ payloads.clear();
+ packet_infos.clear();
+ } else {
+ max_nack_count = std::max(max_nack_count, packet->times_nacked);
+ min_recv_time =
+ std::min(min_recv_time, packet->packet_info.receive_time_ms());
+ max_recv_time =
+ std::max(max_recv_time, packet->packet_info.receive_time_ms());
+ }
+ payloads.emplace_back(packet->video_payload);
+ packet_infos.push_back(packet->packet_info);
+
+ frame_boundary = packet->is_last_packet_in_frame();
+ if (packet->is_last_packet_in_frame()) {
+ auto depacketizer_it = payload_type_map_.find(first_packet->payload_type);
+ RTC_CHECK(depacketizer_it != payload_type_map_.end());
+
+ rtc::scoped_refptr<EncodedImageBuffer> bitstream =
+ depacketizer_it->second->AssembleFrame(payloads);
+ if (!bitstream) {
+ // Failed to assemble a frame. Discard and continue.
+ continue;
+ }
+
+ const video_coding::PacketBuffer::Packet& last_packet = *packet;
+ OnAssembledFrame(std::make_unique<video_coding::RtpFrameObject>(
+ first_packet->seq_num, //
+ last_packet.seq_num, //
+ last_packet.marker_bit, //
+ max_nack_count, //
+ min_recv_time, //
+ max_recv_time, //
+ first_packet->timestamp, //
+ first_packet->ntp_time_ms, //
+ last_packet.video_header.video_timing, //
+ first_packet->payload_type, //
+ first_packet->codec(), //
+ last_packet.video_header.rotation, //
+ last_packet.video_header.content_type, //
+ first_packet->video_header, //
+ last_packet.video_header.color_space, //
+ RtpPacketInfos(std::move(packet_infos)), //
+ std::move(bitstream)));
+ }
+ }
+ RTC_DCHECK(frame_boundary);
+ if (result.buffer_cleared) {
+ RequestKeyFrame();
+ }
+}
+
+void RtpVideoStreamReceiver2::OnAssembledFrame(
+ std::unique_ptr<video_coding::RtpFrameObject> frame) {
+ RTC_DCHECK_RUN_ON(&network_tc_);
+ RTC_DCHECK(frame);
+
+ const absl::optional<RTPVideoHeader::GenericDescriptorInfo>& descriptor =
+ frame->GetRtpVideoHeader().generic;
+
+ if (loss_notification_controller_ && descriptor) {
+ loss_notification_controller_->OnAssembledFrame(
+ frame->first_seq_num(), descriptor->frame_id,
+ absl::c_linear_search(descriptor->decode_target_indications,
+ DecodeTargetIndication::kDiscardable),
+ descriptor->dependencies);
+ }
+
+ // If frames arrive before a key frame, they would not be decodable.
+ // In that case, request a key frame ASAP.
+ if (!has_received_frame_) {
+ if (frame->FrameType() != VideoFrameType::kVideoFrameKey) {
+ // |loss_notification_controller_|, if present, would have already
+ // requested a key frame when the first packet for the non-key frame
+ // had arrived, so no need to replicate the request.
+ if (!loss_notification_controller_) {
+ RequestKeyFrame();
+ }
+ }
+ has_received_frame_ = true;
+ }
+
+ rtc::CritScope lock(&reference_finder_lock_);
+ // Reset |reference_finder_| if |frame| is new and the codec have changed.
+ if (current_codec_) {
+ bool frame_is_newer =
+ AheadOf(frame->Timestamp(), last_assembled_frame_rtp_timestamp_);
+
+ if (frame->codec_type() != current_codec_) {
+ if (frame_is_newer) {
+ // When we reset the |reference_finder_| we don't want new picture ids
+ // to overlap with old picture ids. To ensure that doesn't happen we
+ // start from the |last_completed_picture_id_| and add an offset in case
+ // of reordering.
+ reference_finder_ =
+ std::make_unique<video_coding::RtpFrameReferenceFinder>(
+ this, last_completed_picture_id_ +
+ std::numeric_limits<uint16_t>::max());
+ current_codec_ = frame->codec_type();
+ } else {
+ // Old frame from before the codec switch, discard it.
+ return;
+ }
+ }
+
+ if (frame_is_newer) {
+ last_assembled_frame_rtp_timestamp_ = frame->Timestamp();
+ }
+ } else {
+ current_codec_ = frame->codec_type();
+ last_assembled_frame_rtp_timestamp_ = frame->Timestamp();
+ }
+
+ if (buffered_frame_decryptor_ != nullptr) {
+ buffered_frame_decryptor_->ManageEncryptedFrame(std::move(frame));
+ } else if (frame_transformer_delegate_) {
+ frame_transformer_delegate_->TransformFrame(std::move(frame));
+ } else {
+ reference_finder_->ManageFrame(std::move(frame));
+ }
+}
+
+void RtpVideoStreamReceiver2::OnCompleteFrame(
+ std::unique_ptr<video_coding::EncodedFrame> frame) {
+ {
+ rtc::CritScope lock(&last_seq_num_cs_);
+ video_coding::RtpFrameObject* rtp_frame =
+ static_cast<video_coding::RtpFrameObject*>(frame.get());
+ last_seq_num_for_pic_id_[rtp_frame->id.picture_id] =
+ rtp_frame->last_seq_num();
+ }
+ last_completed_picture_id_ =
+ std::max(last_completed_picture_id_, frame->id.picture_id);
+ complete_frame_callback_->OnCompleteFrame(std::move(frame));
+}
+
+void RtpVideoStreamReceiver2::OnDecryptedFrame(
+ std::unique_ptr<video_coding::RtpFrameObject> frame) {
+ rtc::CritScope lock(&reference_finder_lock_);
+ reference_finder_->ManageFrame(std::move(frame));
+}
+
+void RtpVideoStreamReceiver2::OnDecryptionStatusChange(
+ FrameDecryptorInterface::Status status) {
+ frames_decryptable_.store(
+ (status == FrameDecryptorInterface::Status::kOk) ||
+ (status == FrameDecryptorInterface::Status::kRecoverable));
+}
+
+void RtpVideoStreamReceiver2::SetFrameDecryptor(
+ rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) {
+ RTC_DCHECK_RUN_ON(&network_tc_);
+ if (buffered_frame_decryptor_ == nullptr) {
+ buffered_frame_decryptor_ =
+ std::make_unique<BufferedFrameDecryptor>(this, this);
+ }
+ buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor));
+}
+
+void RtpVideoStreamReceiver2::SetDepacketizerToDecoderFrameTransformer(
+ rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {
+ RTC_DCHECK_RUN_ON(&network_tc_);
+ frame_transformer_delegate_ =
+ new rtc::RefCountedObject<RtpVideoStreamReceiverFrameTransformerDelegate>(
+ this, std::move(frame_transformer), rtc::Thread::Current(),
+ config_.rtp.remote_ssrc);
+ frame_transformer_delegate_->Init();
+}
+
+void RtpVideoStreamReceiver2::UpdateRtt(int64_t max_rtt_ms) {
+ if (nack_module_)
+ nack_module_->UpdateRtt(max_rtt_ms);
+}
+
+absl::optional<int64_t> RtpVideoStreamReceiver2::LastReceivedPacketMs() const {
+ return packet_buffer_.LastReceivedPacketMs();
+}
+
+absl::optional<int64_t> RtpVideoStreamReceiver2::LastReceivedKeyframePacketMs()
+ const {
+ return packet_buffer_.LastReceivedKeyframePacketMs();
+}
+
+void RtpVideoStreamReceiver2::AddSecondarySink(RtpPacketSinkInterface* sink) {
+ RTC_DCHECK_RUN_ON(&worker_task_checker_);
+ RTC_DCHECK(!absl::c_linear_search(secondary_sinks_, sink));
+ secondary_sinks_.push_back(sink);
+}
+
+void RtpVideoStreamReceiver2::RemoveSecondarySink(
+ const RtpPacketSinkInterface* sink) {
+ RTC_DCHECK_RUN_ON(&worker_task_checker_);
+ auto it = absl::c_find(secondary_sinks_, sink);
+ if (it == secondary_sinks_.end()) {
+ // We might be rolling-back a call whose setup failed mid-way. In such a
+ // case, it's simpler to remove "everything" rather than remember what
+ // has already been added.
+ RTC_LOG(LS_WARNING) << "Removal of unknown sink.";
+ return;
+ }
+ secondary_sinks_.erase(it);
+}
+
+void RtpVideoStreamReceiver2::ManageFrame(
+ std::unique_ptr<video_coding::RtpFrameObject> frame) {
+ rtc::CritScope lock(&reference_finder_lock_);
+ reference_finder_->ManageFrame(std::move(frame));
+}
+
+void RtpVideoStreamReceiver2::ReceivePacket(const RtpPacketReceived& packet) {
+ if (packet.payload_size() == 0) {
+ // Padding or keep-alive packet.
+ // TODO(nisse): Could drop empty packets earlier, but need to figure out how
+ // they should be counted in stats.
+ NotifyReceiverOfEmptyPacket(packet.SequenceNumber());
+ return;
+ }
+ if (packet.PayloadType() == config_.rtp.red_payload_type) {
+ ParseAndHandleEncapsulatingHeader(packet);
+ return;
+ }
+
+ const auto type_it = payload_type_map_.find(packet.PayloadType());
+ if (type_it == payload_type_map_.end()) {
+ return;
+ }
+ absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed_payload =
+ type_it->second->Parse(packet.PayloadBuffer());
+ if (parsed_payload == absl::nullopt) {
+ RTC_LOG(LS_WARNING) << "Failed parsing payload.";
+ return;
+ }
+
+ OnReceivedPayloadData(std::move(parsed_payload->video_payload), packet,
+ parsed_payload->video_header);
+}
+
+void RtpVideoStreamReceiver2::ParseAndHandleEncapsulatingHeader(
+ const RtpPacketReceived& packet) {
+ RTC_DCHECK_RUN_ON(&worker_task_checker_);
+ if (packet.PayloadType() == config_.rtp.red_payload_type &&
+ packet.payload_size() > 0) {
+ if (packet.payload()[0] == config_.rtp.ulpfec_payload_type) {
+ // Notify video_receiver about received FEC packets to avoid NACKing these
+ // packets.
+ NotifyReceiverOfEmptyPacket(packet.SequenceNumber());
+ }
+ if (!ulpfec_receiver_->AddReceivedRedPacket(
+ packet, config_.rtp.ulpfec_payload_type)) {
+ return;
+ }
+ ulpfec_receiver_->ProcessReceivedFec();
+ }
+}
+
+// In the case of a video stream without picture ids and no rtx the
+// RtpFrameReferenceFinder will need to know about padding to
+// correctly calculate frame references.
+void RtpVideoStreamReceiver2::NotifyReceiverOfEmptyPacket(uint16_t seq_num) {
+ {
+ rtc::CritScope lock(&reference_finder_lock_);
+ reference_finder_->PaddingReceived(seq_num);
+ }
+ OnInsertedPacket(packet_buffer_.InsertPadding(seq_num));
+ if (nack_module_) {
+ nack_module_->OnReceivedPacket(seq_num, /* is_keyframe = */ false,
+ /* is _recovered = */ false);
+ }
+ if (loss_notification_controller_) {
+ // TODO(bugs.webrtc.org/10336): Handle empty packets.
+ RTC_LOG(LS_WARNING)
+ << "LossNotificationController does not expect empty packets.";
+ }
+}
+
+bool RtpVideoStreamReceiver2::DeliverRtcp(const uint8_t* rtcp_packet,
+ size_t rtcp_packet_length) {
+ RTC_DCHECK_RUN_ON(&worker_task_checker_);
+
+ if (!receiving_) {
+ return false;
+ }
+
+ rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length);
+
+ int64_t rtt = 0;
+ rtp_rtcp_->RTT(config_.rtp.remote_ssrc, &rtt, nullptr, nullptr, nullptr);
+ if (rtt == 0) {
+ // Waiting for valid rtt.
+ return true;
+ }
+ uint32_t ntp_secs = 0;
+ uint32_t ntp_frac = 0;
+ uint32_t rtp_timestamp = 0;
+ uint32_t recieved_ntp_secs = 0;
+ uint32_t recieved_ntp_frac = 0;
+ if (rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, &recieved_ntp_secs,
+ &recieved_ntp_frac, &rtp_timestamp) != 0) {
+ // Waiting for RTCP.
+ return true;
+ }
+ NtpTime recieved_ntp(recieved_ntp_secs, recieved_ntp_frac);
+ int64_t time_since_recieved =
+ clock_->CurrentNtpInMilliseconds() - recieved_ntp.ToMs();
+ // Don't use old SRs to estimate time.
+ if (time_since_recieved <= 1) {
+ ntp_estimator_.UpdateRtcpTimestamp(rtt, ntp_secs, ntp_frac, rtp_timestamp);
+ absl::optional<int64_t> remote_to_local_clock_offset_ms =
+ ntp_estimator_.EstimateRemoteToLocalClockOffsetMs();
+ if (remote_to_local_clock_offset_ms.has_value()) {
+ absolute_capture_time_receiver_.SetRemoteToLocalClockOffset(
+ Int64MsToQ32x32(*remote_to_local_clock_offset_ms));
+ }
+ }
+
+ return true;
+}
+
+void RtpVideoStreamReceiver2::FrameContinuous(int64_t picture_id) {
+ if (!nack_module_)
+ return;
+
+ int seq_num = -1;
+ {
+ rtc::CritScope lock(&last_seq_num_cs_);
+ auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id);
+ if (seq_num_it != last_seq_num_for_pic_id_.end())
+ seq_num = seq_num_it->second;
+ }
+ if (seq_num != -1)
+ nack_module_->ClearUpTo(seq_num);
+}
+
+void RtpVideoStreamReceiver2::FrameDecoded(int64_t picture_id) {
+ int seq_num = -1;
+ {
+ rtc::CritScope lock(&last_seq_num_cs_);
+ auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id);
+ if (seq_num_it != last_seq_num_for_pic_id_.end()) {
+ seq_num = seq_num_it->second;
+ last_seq_num_for_pic_id_.erase(last_seq_num_for_pic_id_.begin(),
+ ++seq_num_it);
+ }
+ }
+ if (seq_num != -1) {
+ packet_buffer_.ClearTo(seq_num);
+ rtc::CritScope lock(&reference_finder_lock_);
+ reference_finder_->ClearTo(seq_num);
+ }
+}
+
+void RtpVideoStreamReceiver2::SignalNetworkState(NetworkState state) {
+ rtp_rtcp_->SetRTCPStatus(state == kNetworkUp ? config_.rtp.rtcp_mode
+ : RtcpMode::kOff);
+}
+
+void RtpVideoStreamReceiver2::StartReceive() {
+ RTC_DCHECK_RUN_ON(&worker_task_checker_);
+ receiving_ = true;
+}
+
+void RtpVideoStreamReceiver2::StopReceive() {
+ RTC_DCHECK_RUN_ON(&worker_task_checker_);
+ receiving_ = false;
+}
+
+void RtpVideoStreamReceiver2::UpdateHistograms() {
+ FecPacketCounter counter = ulpfec_receiver_->GetPacketCounter();
+ if (counter.first_packet_time_ms == -1)
+ return;
+
+ int64_t elapsed_sec =
+ (clock_->TimeInMilliseconds() - counter.first_packet_time_ms) / 1000;
+ if (elapsed_sec < metrics::kMinRunTimeInSeconds)
+ return;
+
+ if (counter.num_packets > 0) {
+ RTC_HISTOGRAM_PERCENTAGE(
+ "WebRTC.Video.ReceivedFecPacketsInPercent",
+ static_cast<int>(counter.num_fec_packets * 100 / counter.num_packets));
+ }
+ if (counter.num_fec_packets > 0) {
+ RTC_HISTOGRAM_PERCENTAGE("WebRTC.Video.RecoveredMediaPacketsInPercentOfFec",
+ static_cast<int>(counter.num_recovered_packets *
+ 100 / counter.num_fec_packets));
+ }
+ if (config_.rtp.ulpfec_payload_type != -1) {
+ RTC_HISTOGRAM_COUNTS_10000(
+ "WebRTC.Video.FecBitrateReceivedInKbps",
+ static_cast<int>(counter.num_bytes * 8 / elapsed_sec / 1000));
+ }
+}
+
+void RtpVideoStreamReceiver2::InsertSpsPpsIntoTracker(uint8_t payload_type) {
+ auto codec_params_it = pt_codec_params_.find(payload_type);
+ if (codec_params_it == pt_codec_params_.end())
+ return;
+
+ RTC_LOG(LS_INFO) << "Found out of band supplied codec parameters for"
+ " payload type: "
+ << static_cast<int>(payload_type);
+
+ H264SpropParameterSets sprop_decoder;
+ auto sprop_base64_it =
+ codec_params_it->second.find(cricket::kH264FmtpSpropParameterSets);
+
+ if (sprop_base64_it == codec_params_it->second.end())
+ return;
+
+ if (!sprop_decoder.DecodeSprop(sprop_base64_it->second.c_str()))
+ return;
+
+ tracker_.InsertSpsPpsNalus(sprop_decoder.sps_nalu(),
+ sprop_decoder.pps_nalu());
+}
+
+} // namespace webrtc
diff --git a/video/rtp_video_stream_receiver2.h b/video/rtp_video_stream_receiver2.h
new file mode 100644
index 0000000..3026e1d
--- /dev/null
+++ b/video/rtp_video_stream_receiver2.h
@@ -0,0 +1,386 @@
+/*
+ * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_
+#define VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_
+
+#include <atomic>
+#include <list>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/array_view.h"
+#include "api/crypto/frame_decryptor_interface.h"
+#include "api/video/color_space.h"
+#include "api/video_codecs/video_codec.h"
+#include "call/rtp_packet_sink_interface.h"
+#include "call/syncable.h"
+#include "call/video_receive_stream.h"
+#include "modules/rtp_rtcp/include/receive_statistics.h"
+#include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h"
+#include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
+#include "modules/rtp_rtcp/include/rtp_rtcp.h"
+#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
+#include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h"
+#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
+#include "modules/rtp_rtcp/source/rtp_packet_received.h"
+#include "modules/rtp_rtcp/source/rtp_video_header.h"
+#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h"
+#include "modules/video_coding/h264_sps_pps_tracker.h"
+#include "modules/video_coding/loss_notification_controller.h"
+#include "modules/video_coding/packet_buffer.h"
+#include "modules/video_coding/rtp_frame_reference_finder.h"
+#include "modules/video_coding/unique_timestamp_counter.h"
+#include "rtc_base/constructor_magic.h"
+#include "rtc_base/critical_section.h"
+#include "rtc_base/experiments/field_trial_parser.h"
+#include "rtc_base/numerics/sequence_number_util.h"
+#include "rtc_base/synchronization/sequence_checker.h"
+#include "rtc_base/thread_annotations.h"
+#include "rtc_base/thread_checker.h"
+#include "video/buffered_frame_decryptor.h"
+#include "video/rtp_video_stream_receiver_frame_transformer_delegate.h"
+
+namespace webrtc {
+
+class NackModule2;
+class PacketRouter;
+class ProcessThread;
+class ReceiveStatistics;
+class RtcpRttStats;
+class RtpPacketReceived;
+class Transport;
+class UlpfecReceiver;
+
+class RtpVideoStreamReceiver2 : public LossNotificationSender,
+ public RecoveredPacketReceiver,
+ public RtpPacketSinkInterface,
+ public KeyFrameRequestSender,
+ public video_coding::OnCompleteFrameCallback,
+ public OnDecryptedFrameCallback,
+ public OnDecryptionStatusChangeCallback,
+ public RtpVideoFrameReceiver {
+ public:
+ RtpVideoStreamReceiver2(
+ Clock* clock,
+ Transport* transport,
+ RtcpRttStats* rtt_stats,
+ // The packet router is optional; if provided, the RtpRtcp module for this
+ // stream is registered as a candidate for sending REMB and transport
+ // feedback.
+ PacketRouter* packet_router,
+ const VideoReceiveStream::Config* config,
+ ReceiveStatistics* rtp_receive_statistics,
+ RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
+ RtcpCnameCallback* rtcp_cname_callback,
+ ProcessThread* process_thread,
+ NackSender* nack_sender,
+ // The KeyFrameRequestSender is optional; if not provided, key frame
+ // requests are sent via the internal RtpRtcp module.
+ KeyFrameRequestSender* keyframe_request_sender,
+ video_coding::OnCompleteFrameCallback* complete_frame_callback,
+ rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
+ rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
+ ~RtpVideoStreamReceiver2() override;
+
+ void AddReceiveCodec(const VideoCodec& video_codec,
+ const std::map<std::string, std::string>& codec_params,
+ bool raw_payload);
+
+ void StartReceive();
+ void StopReceive();
+
+ // Produces the transport-related timestamps; current_delay_ms is left unset.
+ absl::optional<Syncable::Info> GetSyncInfo() const;
+
+ bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length);
+
+ void FrameContinuous(int64_t seq_num);
+
+ void FrameDecoded(int64_t seq_num);
+
+ void SignalNetworkState(NetworkState state);
+
+ // Returns number of different frames seen.
+ int GetUniqueFramesSeen() const {
+ RTC_DCHECK_RUN_ON(&worker_task_checker_);
+ return frame_counter_.GetUniqueSeen();
+ }
+
+ // Implements RtpPacketSinkInterface.
+ void OnRtpPacket(const RtpPacketReceived& packet) override;
+
+ // TODO(philipel): Stop using VCMPacket in the new jitter buffer and then
+ // remove this function. Public only for tests.
+ void OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload,
+ const RtpPacketReceived& rtp_packet,
+ const RTPVideoHeader& video);
+
+ // Implements RecoveredPacketReceiver.
+ void OnRecoveredPacket(const uint8_t* packet, size_t packet_length) override;
+
+ // Send an RTCP keyframe request.
+ void RequestKeyFrame() override;
+
+ // Implements LossNotificationSender.
+ void SendLossNotification(uint16_t last_decoded_seq_num,
+ uint16_t last_received_seq_num,
+ bool decodability_flag,
+ bool buffering_allowed) override;
+
+ bool IsUlpfecEnabled() const;
+ bool IsRetransmissionsEnabled() const;
+
+ // Returns true if a decryptor is attached and frames can be decrypted.
+ // Updated by OnDecryptionStatusChangeCallback. Note this refers to Frame
+ // Decryption not SRTP.
+ bool IsDecryptable() const;
+
+ // Don't use, still experimental.
+ void RequestPacketRetransmit(const std::vector<uint16_t>& sequence_numbers);
+
+ // Implements OnCompleteFrameCallback.
+ void OnCompleteFrame(
+ std::unique_ptr<video_coding::EncodedFrame> frame) override;
+
+ // Implements OnDecryptedFrameCallback.
+ void OnDecryptedFrame(
+ std::unique_ptr<video_coding::RtpFrameObject> frame) override;
+
+ // Implements OnDecryptionStatusChangeCallback.
+ void OnDecryptionStatusChange(
+ FrameDecryptorInterface::Status status) override;
+
+ // Optionally set a frame decryptor after a stream has started. This will not
+ // reset the decoder state.
+ void SetFrameDecryptor(
+ rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor);
+
+ // Sets a frame transformer after a stream has started, if no transformer
+ // has previously been set. Does not reset the decoder state.
+ void SetDepacketizerToDecoderFrameTransformer(
+ rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
+
+ // Called by VideoReceiveStream when stats are updated.
+ void UpdateRtt(int64_t max_rtt_ms);
+
+ absl::optional<int64_t> LastReceivedPacketMs() const;
+ absl::optional<int64_t> LastReceivedKeyframePacketMs() const;
+
+ // RtpDemuxer only forwards a given RTP packet to one sink. However, some
+ // sinks, such as FlexFEC, might wish to be informed of all of the packets
+ // a given sink receives (or any set of sinks). They may do so by registering
+ // themselves as secondary sinks.
+ void AddSecondarySink(RtpPacketSinkInterface* sink);
+ void RemoveSecondarySink(const RtpPacketSinkInterface* sink);
+
+ private:
+ // Implements RtpVideoFrameReceiver.
+ void ManageFrame(
+ std::unique_ptr<video_coding::RtpFrameObject> frame) override;
+
+ // Used for buffering RTCP feedback messages and sending them all together.
+ // Note:
+ // 1. Key frame requests and NACKs are mutually exclusive, with the
+ // former taking precedence over the latter.
+ // 2. Loss notifications are orthogonal to either. (That is, may be sent
+ // alongside either.)
+ class RtcpFeedbackBuffer : public KeyFrameRequestSender,
+ public NackSender,
+ public LossNotificationSender {
+ public:
+ RtcpFeedbackBuffer(KeyFrameRequestSender* key_frame_request_sender,
+ NackSender* nack_sender,
+ LossNotificationSender* loss_notification_sender);
+
+ ~RtcpFeedbackBuffer() override = default;
+
+ // KeyFrameRequestSender implementation.
+ void RequestKeyFrame() RTC_LOCKS_EXCLUDED(cs_) override;
+
+ // NackSender implementation.
+ void SendNack(const std::vector<uint16_t>& sequence_numbers,
+ bool buffering_allowed) RTC_LOCKS_EXCLUDED(cs_) override;
+
+ // LossNotificationSender implementation.
+ void SendLossNotification(uint16_t last_decoded_seq_num,
+ uint16_t last_received_seq_num,
+ bool decodability_flag,
+ bool buffering_allowed)
+ RTC_LOCKS_EXCLUDED(cs_) override;
+
+ // Send all RTCP feedback messages buffered thus far.
+ void SendBufferedRtcpFeedback() RTC_LOCKS_EXCLUDED(cs_);
+
+ private:
+ // LNTF-related state.
+ struct LossNotificationState {
+ LossNotificationState(uint16_t last_decoded_seq_num,
+ uint16_t last_received_seq_num,
+ bool decodability_flag)
+ : last_decoded_seq_num(last_decoded_seq_num),
+ last_received_seq_num(last_received_seq_num),
+ decodability_flag(decodability_flag) {}
+
+ uint16_t last_decoded_seq_num;
+ uint16_t last_received_seq_num;
+ bool decodability_flag;
+ };
+ struct ConsumedRtcpFeedback {
+ bool request_key_frame = false;
+ std::vector<uint16_t> nack_sequence_numbers;
+ absl::optional<LossNotificationState> lntf_state;
+ };
+
+ ConsumedRtcpFeedback ConsumeRtcpFeedback() RTC_LOCKS_EXCLUDED(cs_);
+ ConsumedRtcpFeedback ConsumeRtcpFeedbackLocked()
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(cs_);
+ // This method is called both with and without cs_ held.
+ void SendRtcpFeedback(ConsumedRtcpFeedback feedback);
+
+ KeyFrameRequestSender* const key_frame_request_sender_;
+ NackSender* const nack_sender_;
+ LossNotificationSender* const loss_notification_sender_;
+
+ // NACKs are accessible from two threads due to nack_module_ being a module.
+ rtc::CriticalSection cs_;
+
+ // Key-frame-request-related state.
+ bool request_key_frame_ RTC_GUARDED_BY(cs_);
+
+ // NACK-related state.
+ std::vector<uint16_t> nack_sequence_numbers_ RTC_GUARDED_BY(cs_);
+
+ absl::optional<LossNotificationState> lntf_state_ RTC_GUARDED_BY(cs_);
+ };
+ enum ParseGenericDependenciesResult {
+ kDropPacket,
+ kHasGenericDescriptor,
+ kNoGenericDescriptor
+ };
+
+ // Entry point doing non-stats work for a received packet. Called
+ // for the same packet both before and after RED decapsulation.
+ void ReceivePacket(const RtpPacketReceived& packet);
+ // Parses and handles RED headers.
+ // This function assumes that it's being called from only one thread.
+ void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet);
+ void NotifyReceiverOfEmptyPacket(uint16_t seq_num);
+ void UpdateHistograms();
+ bool IsRedEnabled() const;
+ void InsertSpsPpsIntoTracker(uint8_t payload_type);
+ void OnInsertedPacket(video_coding::PacketBuffer::InsertResult result);
+ ParseGenericDependenciesResult ParseGenericDependenciesExtension(
+ const RtpPacketReceived& rtp_packet,
+ RTPVideoHeader* video_header) RTC_RUN_ON(worker_task_checker_);
+ void OnAssembledFrame(std::unique_ptr<video_coding::RtpFrameObject> frame);
+
+ Clock* const clock_;
+ // Ownership of this object lies with VideoReceiveStream, which owns |this|.
+ const VideoReceiveStream::Config& config_;
+ PacketRouter* const packet_router_;
+ ProcessThread* const process_thread_;
+
+ RemoteNtpTimeEstimator ntp_estimator_;
+
+ RtpHeaderExtensionMap rtp_header_extensions_;
+ // Set by the field trial WebRTC-ForcePlayoutDelay to override any playout
+ // delay that is specified in the received packets.
+ FieldTrialOptional<int> forced_playout_delay_max_ms_;
+ FieldTrialOptional<int> forced_playout_delay_min_ms_;
+ ReceiveStatistics* const rtp_receive_statistics_;
+ std::unique_ptr<UlpfecReceiver> ulpfec_receiver_;
+
+ SequenceChecker worker_task_checker_;
+ bool receiving_ RTC_GUARDED_BY(worker_task_checker_);
+ int64_t last_packet_log_ms_ RTC_GUARDED_BY(worker_task_checker_);
+
+ const std::unique_ptr<RtpRtcp> rtp_rtcp_;
+
+ video_coding::OnCompleteFrameCallback* complete_frame_callback_;
+ KeyFrameRequestSender* const keyframe_request_sender_;
+
+ RtcpFeedbackBuffer rtcp_feedback_buffer_;
+ std::unique_ptr<NackModule2> nack_module_;
+ std::unique_ptr<LossNotificationController> loss_notification_controller_;
+
+ video_coding::PacketBuffer packet_buffer_;
+ UniqueTimestampCounter frame_counter_ RTC_GUARDED_BY(worker_task_checker_);
+ SeqNumUnwrapper<uint16_t> frame_id_unwrapper_
+ RTC_GUARDED_BY(worker_task_checker_);
+
+ // Video structure provided in the dependency descriptor in a first packet
+ // of a key frame. It is required to parse dependency descriptor in the
+ // following delta packets.
+ std::unique_ptr<FrameDependencyStructure> video_structure_
+ RTC_GUARDED_BY(worker_task_checker_);
+ // Frame id of the last frame with the attached video structure.
+ // absl::nullopt when `video_structure_ == nullptr`;
+ absl::optional<int64_t> video_structure_frame_id_
+ RTC_GUARDED_BY(worker_task_checker_);
+
+ rtc::CriticalSection reference_finder_lock_;
+ std::unique_ptr<video_coding::RtpFrameReferenceFinder> reference_finder_
+ RTC_GUARDED_BY(reference_finder_lock_);
+ absl::optional<VideoCodecType> current_codec_;
+ uint32_t last_assembled_frame_rtp_timestamp_;
+
+ rtc::CriticalSection last_seq_num_cs_;
+ std::map<int64_t, uint16_t> last_seq_num_for_pic_id_
+ RTC_GUARDED_BY(last_seq_num_cs_);
+ video_coding::H264SpsPpsTracker tracker_;
+
+ // Maps payload id to the depacketizer.
+ std::map<uint8_t, std::unique_ptr<VideoRtpDepacketizer>> payload_type_map_;
+
+ // TODO(johan): Remove pt_codec_params_ once
+ // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved.
+ // Maps a payload type to a map of out-of-band supplied codec parameters.
+ std::map<uint8_t, std::map<std::string, std::string>> pt_codec_params_;
+ int16_t last_payload_type_ = -1;
+
+ bool has_received_frame_;
+
+ std::vector<RtpPacketSinkInterface*> secondary_sinks_
+ RTC_GUARDED_BY(worker_task_checker_);
+
+ // Info for GetSyncInfo is updated on network or worker thread, and queried on
+ // the worker thread.
+ rtc::CriticalSection sync_info_lock_;
+ absl::optional<uint32_t> last_received_rtp_timestamp_
+ RTC_GUARDED_BY(sync_info_lock_);
+ absl::optional<int64_t> last_received_rtp_system_time_ms_
+ RTC_GUARDED_BY(sync_info_lock_);
+
+ // Used to validate the buffered frame decryptor is always run on the correct
+ // thread.
+ rtc::ThreadChecker network_tc_;
+ // Handles incoming encrypted frames and forwards them to the
+ // rtp_reference_finder if they are decryptable.
+ std::unique_ptr<BufferedFrameDecryptor> buffered_frame_decryptor_
+ RTC_PT_GUARDED_BY(network_tc_);
+ std::atomic<bool> frames_decryptable_;
+ absl::optional<ColorSpace> last_color_space_;
+
+ AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_
+ RTC_GUARDED_BY(worker_task_checker_);
+
+ int64_t last_completed_picture_id_ = 0;
+
+ rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate>
+ frame_transformer_delegate_;
+};
+
+} // namespace webrtc
+
+#endif // VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_
diff --git a/video/rtp_video_stream_receiver2_unittest.cc b/video/rtp_video_stream_receiver2_unittest.cc
new file mode 100644
index 0000000..c8584fc
--- /dev/null
+++ b/video/rtp_video_stream_receiver2_unittest.cc
@@ -0,0 +1,1215 @@
+/*
+ * Copyright 2017 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/rtp_video_stream_receiver2.h"
+
+#include <memory>
+#include <utility>
+
+#include "api/video/video_codec_type.h"
+#include "api/video/video_frame_type.h"
+#include "common_video/h264/h264_common.h"
+#include "media/base/media_constants.h"
+#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
+#include "modules/rtp_rtcp/source/rtp_format.h"
+#include "modules/rtp_rtcp/source/rtp_format_vp9.h"
+#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h"
+#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
+#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
+#include "modules/rtp_rtcp/source/rtp_packet_received.h"
+#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
+#include "modules/utility/include/process_thread.h"
+#include "modules/video_coding/frame_object.h"
+#include "modules/video_coding/include/video_coding_defines.h"
+#include "modules/video_coding/rtp_frame_reference_finder.h"
+#include "rtc_base/byte_buffer.h"
+#include "rtc_base/logging.h"
+#include "system_wrappers/include/clock.h"
+#include "system_wrappers/include/field_trial.h"
+#include "test/field_trial.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/mock_frame_transformer.h"
+
+using ::testing::_;
+using ::testing::ElementsAre;
+using ::testing::Invoke;
+using ::testing::SizeIs;
+using ::testing::Values;
+
+namespace webrtc {
+
+namespace {
+
+const uint8_t kH264StartCode[] = {0x00, 0x00, 0x00, 0x01};
+
+std::vector<uint64_t> GetAbsoluteCaptureTimestamps(
+ const video_coding::EncodedFrame* frame) {
+ std::vector<uint64_t> result;
+ for (const auto& packet_info : frame->PacketInfos()) {
+ if (packet_info.absolute_capture_time()) {
+ result.push_back(
+ packet_info.absolute_capture_time()->absolute_capture_timestamp);
+ }
+ }
+ return result;
+}
+
+RTPVideoHeader GetGenericVideoHeader(VideoFrameType frame_type) {
+ RTPVideoHeader video_header;
+ video_header.is_first_packet_in_frame = true;
+ video_header.is_last_packet_in_frame = true;
+ video_header.codec = kVideoCodecGeneric;
+ video_header.frame_type = frame_type;
+ return video_header;
+}
+
+class MockTransport : public Transport {
+ public:
+ MOCK_METHOD(bool,
+ SendRtp,
+ (const uint8_t*, size_t length, const PacketOptions& options),
+ (override));
+ MOCK_METHOD(bool, SendRtcp, (const uint8_t*, size_t length), (override));
+};
+
+class MockNackSender : public NackSender {
+ public:
+ MOCK_METHOD(void,
+ SendNack,
+ (const std::vector<uint16_t>& sequence_numbers,
+ bool buffering_allowed),
+ (override));
+};
+
+class MockKeyFrameRequestSender : public KeyFrameRequestSender {
+ public:
+ MOCK_METHOD(void, RequestKeyFrame, (), (override));
+};
+
+class MockOnCompleteFrameCallback
+ : public video_coding::OnCompleteFrameCallback {
+ public:
+ MOCK_METHOD(void, DoOnCompleteFrame, (video_coding::EncodedFrame*), ());
+ MOCK_METHOD(void,
+ DoOnCompleteFrameFailNullptr,
+ (video_coding::EncodedFrame*),
+ ());
+ MOCK_METHOD(void,
+ DoOnCompleteFrameFailLength,
+ (video_coding::EncodedFrame*),
+ ());
+ MOCK_METHOD(void,
+ DoOnCompleteFrameFailBitstream,
+ (video_coding::EncodedFrame*),
+ ());
+ void OnCompleteFrame(
+ std::unique_ptr<video_coding::EncodedFrame> frame) override {
+ if (!frame) {
+ DoOnCompleteFrameFailNullptr(nullptr);
+ return;
+ }
+ EXPECT_EQ(buffer_.Length(), frame->size());
+ if (buffer_.Length() != frame->size()) {
+ DoOnCompleteFrameFailLength(frame.get());
+ return;
+ }
+ if (frame->size() != buffer_.Length() ||
+ memcmp(buffer_.Data(), frame->data(), buffer_.Length()) != 0) {
+ DoOnCompleteFrameFailBitstream(frame.get());
+ return;
+ }
+ DoOnCompleteFrame(frame.get());
+ }
+
+ void ClearExpectedBitstream() { buffer_.Clear(); }
+
+ void AppendExpectedBitstream(const uint8_t data[], size_t size_in_bytes) {
+ // TODO(Johan): Let rtc::ByteBuffer handle uint8_t* instead of char*.
+ buffer_.WriteBytes(reinterpret_cast<const char*>(data), size_in_bytes);
+ }
+ rtc::ByteBufferWriter buffer_;
+};
+
+class MockRtpPacketSink : public RtpPacketSinkInterface {
+ public:
+ MOCK_METHOD(void, OnRtpPacket, (const RtpPacketReceived&), (override));
+};
+
+constexpr uint32_t kSsrc = 111;
+constexpr uint16_t kSequenceNumber = 222;
+constexpr int kPayloadType = 100;
+constexpr int kRedPayloadType = 125;
+
+std::unique_ptr<RtpPacketReceived> CreateRtpPacketReceived() {
+ auto packet = std::make_unique<RtpPacketReceived>();
+ packet->SetSsrc(kSsrc);
+ packet->SetSequenceNumber(kSequenceNumber);
+ packet->SetPayloadType(kPayloadType);
+ return packet;
+}
+
+MATCHER_P(SamePacketAs, other, "") {
+ return arg.Ssrc() == other.Ssrc() &&
+ arg.SequenceNumber() == other.SequenceNumber();
+}
+
+} // namespace
+
+class RtpVideoStreamReceiver2Test : public ::testing::Test {
+ public:
+ RtpVideoStreamReceiver2Test() : RtpVideoStreamReceiver2Test("") {}
+ explicit RtpVideoStreamReceiver2Test(std::string field_trials)
+ : override_field_trials_(field_trials),
+ config_(CreateConfig()),
+ process_thread_(ProcessThread::Create("TestThread")) {
+ rtp_receive_statistics_ =
+ ReceiveStatistics::Create(Clock::GetRealTimeClock());
+ rtp_video_stream_receiver_ = std::make_unique<RtpVideoStreamReceiver2>(
+ Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_,
+ rtp_receive_statistics_.get(), nullptr, nullptr, process_thread_.get(),
+ &mock_nack_sender_, &mock_key_frame_request_sender_,
+ &mock_on_complete_frame_callback_, nullptr, nullptr);
+ VideoCodec codec;
+ codec.plType = kPayloadType;
+ codec.codecType = kVideoCodecGeneric;
+ rtp_video_stream_receiver_->AddReceiveCodec(codec, {},
+ /*raw_payload=*/false);
+ }
+
+ RTPVideoHeader GetDefaultH264VideoHeader() {
+ RTPVideoHeader video_header;
+ video_header.codec = kVideoCodecH264;
+ video_header.video_type_header.emplace<RTPVideoHeaderH264>();
+ return video_header;
+ }
+
+ // TODO(Johan): refactor h264_sps_pps_tracker_unittests.cc to avoid duplicate
+ // code.
+ void AddSps(RTPVideoHeader* video_header,
+ uint8_t sps_id,
+ rtc::CopyOnWriteBuffer* data) {
+ NaluInfo info;
+ info.type = H264::NaluType::kSps;
+ info.sps_id = sps_id;
+ info.pps_id = -1;
+ data->AppendData({H264::NaluType::kSps, sps_id});
+ auto& h264 = absl::get<RTPVideoHeaderH264>(video_header->video_type_header);
+ h264.nalus[h264.nalus_length++] = info;
+ }
+
+ void AddPps(RTPVideoHeader* video_header,
+ uint8_t sps_id,
+ uint8_t pps_id,
+ rtc::CopyOnWriteBuffer* data) {
+ NaluInfo info;
+ info.type = H264::NaluType::kPps;
+ info.sps_id = sps_id;
+ info.pps_id = pps_id;
+ data->AppendData({H264::NaluType::kPps, pps_id});
+ auto& h264 = absl::get<RTPVideoHeaderH264>(video_header->video_type_header);
+ h264.nalus[h264.nalus_length++] = info;
+ }
+
+ void AddIdr(RTPVideoHeader* video_header, int pps_id) {
+ NaluInfo info;
+ info.type = H264::NaluType::kIdr;
+ info.sps_id = -1;
+ info.pps_id = pps_id;
+ auto& h264 = absl::get<RTPVideoHeaderH264>(video_header->video_type_header);
+ h264.nalus[h264.nalus_length++] = info;
+ }
+
+ protected:
+ static VideoReceiveStream::Config CreateConfig() {
+ VideoReceiveStream::Config config(nullptr);
+ config.rtp.remote_ssrc = 1111;
+ config.rtp.local_ssrc = 2222;
+ config.rtp.red_payload_type = kRedPayloadType;
+ return config;
+ }
+
+ const webrtc::test::ScopedFieldTrials override_field_trials_;
+ VideoReceiveStream::Config config_;
+ MockNackSender mock_nack_sender_;
+ MockKeyFrameRequestSender mock_key_frame_request_sender_;
+ MockTransport mock_transport_;
+ MockOnCompleteFrameCallback mock_on_complete_frame_callback_;
+ std::unique_ptr<ProcessThread> process_thread_;
+ std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_;
+ std::unique_ptr<RtpVideoStreamReceiver2> rtp_video_stream_receiver_;
+};
+
+TEST_F(RtpVideoStreamReceiver2Test, CacheColorSpaceFromLastPacketOfKeyframe) {
+ // Test that color space is cached from the last packet of a key frame and
+ // that it's not reset by padding packets without color space.
+ constexpr int kVp9PayloadType = 99;
+ const ColorSpace kColorSpace(
+ ColorSpace::PrimaryID::kFILM, ColorSpace::TransferID::kBT2020_12,
+ ColorSpace::MatrixID::kBT2020_NCL, ColorSpace::RangeID::kFull);
+ const std::vector<uint8_t> kKeyFramePayload = {0, 1, 2, 3, 4, 5,
+ 6, 7, 8, 9, 10};
+ const std::vector<uint8_t> kDeltaFramePayload = {0, 1, 2, 3, 4};
+
+ // Anonymous helper class that generates received packets.
+ class {
+ public:
+ void SetPayload(const std::vector<uint8_t>& payload,
+ VideoFrameType video_frame_type) {
+ video_frame_type_ = video_frame_type;
+ RtpPacketizer::PayloadSizeLimits pay_load_size_limits;
+ // Reduce max payload length to make sure the key frame generates two
+ // packets.
+ pay_load_size_limits.max_payload_len = 8;
+ RTPVideoHeaderVP9 rtp_video_header_vp9;
+ rtp_video_header_vp9.InitRTPVideoHeaderVP9();
+ rtp_video_header_vp9.inter_pic_predicted =
+ (video_frame_type == VideoFrameType::kVideoFrameDelta);
+ rtp_packetizer_ = std::make_unique<RtpPacketizerVp9>(
+ payload, pay_load_size_limits, rtp_video_header_vp9);
+ }
+
+ size_t NumPackets() { return rtp_packetizer_->NumPackets(); }
+ void SetColorSpace(const ColorSpace& color_space) {
+ color_space_ = color_space;
+ }
+
+ RtpPacketReceived NextPacket() {
+ RtpHeaderExtensionMap extension_map;
+ extension_map.Register<ColorSpaceExtension>(1);
+ RtpPacketToSend packet_to_send(&extension_map);
+ packet_to_send.SetSequenceNumber(sequence_number_++);
+ packet_to_send.SetSsrc(kSsrc);
+ packet_to_send.SetPayloadType(kVp9PayloadType);
+ bool include_color_space =
+ (rtp_packetizer_->NumPackets() == 1u &&
+ video_frame_type_ == VideoFrameType::kVideoFrameKey);
+ if (include_color_space) {
+ EXPECT_TRUE(
+ packet_to_send.SetExtension<ColorSpaceExtension>(color_space_));
+ }
+ rtp_packetizer_->NextPacket(&packet_to_send);
+
+ RtpPacketReceived received_packet(&extension_map);
+ received_packet.Parse(packet_to_send.data(), packet_to_send.size());
+ return received_packet;
+ }
+
+ private:
+ uint16_t sequence_number_ = 0;
+ VideoFrameType video_frame_type_;
+ ColorSpace color_space_;
+ std::unique_ptr<RtpPacketizer> rtp_packetizer_;
+ } received_packet_generator;
+ received_packet_generator.SetColorSpace(kColorSpace);
+
+ // Prepare the receiver for VP9.
+ VideoCodec codec;
+ codec.plType = kVp9PayloadType;
+ codec.codecType = kVideoCodecVP9;
+ std::map<std::string, std::string> codec_params;
+ rtp_video_stream_receiver_->AddReceiveCodec(codec, codec_params,
+ /*raw_payload=*/false);
+
+ // Generate key frame packets.
+ received_packet_generator.SetPayload(kKeyFramePayload,
+ VideoFrameType::kVideoFrameKey);
+ EXPECT_EQ(received_packet_generator.NumPackets(), 2u);
+ RtpPacketReceived key_frame_packet1 = received_packet_generator.NextPacket();
+ RtpPacketReceived key_frame_packet2 = received_packet_generator.NextPacket();
+
+ // Generate delta frame packet.
+ received_packet_generator.SetPayload(kDeltaFramePayload,
+ VideoFrameType::kVideoFrameDelta);
+ EXPECT_EQ(received_packet_generator.NumPackets(), 1u);
+ RtpPacketReceived delta_frame_packet = received_packet_generator.NextPacket();
+
+ rtp_video_stream_receiver_->StartReceive();
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(
+ kKeyFramePayload.data(), kKeyFramePayload.size());
+
+ // Send the key frame and expect a callback with color space information.
+ EXPECT_FALSE(key_frame_packet1.GetExtension<ColorSpaceExtension>());
+ EXPECT_TRUE(key_frame_packet2.GetExtension<ColorSpaceExtension>());
+ rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet1);
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
+ .WillOnce(Invoke([kColorSpace](video_coding::EncodedFrame* frame) {
+ ASSERT_TRUE(frame->EncodedImage().ColorSpace());
+ EXPECT_EQ(*frame->EncodedImage().ColorSpace(), kColorSpace);
+ }));
+ rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet2);
+ // Resend the first key frame packet to simulate padding for example.
+ rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet1);
+
+ mock_on_complete_frame_callback_.ClearExpectedBitstream();
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(
+ kDeltaFramePayload.data(), kDeltaFramePayload.size());
+
+ // Expect delta frame to have color space set even though color space not
+ // included in the RTP packet.
+ EXPECT_FALSE(delta_frame_packet.GetExtension<ColorSpaceExtension>());
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
+ .WillOnce(Invoke([kColorSpace](video_coding::EncodedFrame* frame) {
+ ASSERT_TRUE(frame->EncodedImage().ColorSpace());
+ EXPECT_EQ(*frame->EncodedImage().ColorSpace(), kColorSpace);
+ }));
+ rtp_video_stream_receiver_->OnRtpPacket(delta_frame_packet);
+}
+
+TEST_F(RtpVideoStreamReceiver2Test, GenericKeyFrame) {
+ RtpPacketReceived rtp_packet;
+ rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
+ rtp_packet.SetPayloadType(kPayloadType);
+ rtp_packet.SetSequenceNumber(1);
+ RTPVideoHeader video_header =
+ GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
+ data.size());
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
+ rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
+ video_header);
+}
+
+TEST_F(RtpVideoStreamReceiver2Test, PacketInfoIsPropagatedIntoVideoFrames) {
+ constexpr uint64_t kAbsoluteCaptureTimestamp = 12;
+ constexpr int kId0 = 1;
+
+ RtpHeaderExtensionMap extension_map;
+ extension_map.Register<AbsoluteCaptureTimeExtension>(kId0);
+ RtpPacketReceived rtp_packet(&extension_map);
+ rtp_packet.SetPayloadType(kPayloadType);
+ rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
+ rtp_packet.SetSequenceNumber(1);
+ rtp_packet.SetTimestamp(1);
+ rtp_packet.SetSsrc(kSsrc);
+ rtp_packet.SetExtension<AbsoluteCaptureTimeExtension>(
+ AbsoluteCaptureTime{kAbsoluteCaptureTimestamp,
+ /*estimated_capture_clock_offset=*/absl::nullopt});
+
+ RTPVideoHeader video_header =
+ GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
+ data.size());
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
+ .WillOnce(Invoke(
+ [kAbsoluteCaptureTimestamp](video_coding::EncodedFrame* frame) {
+ EXPECT_THAT(GetAbsoluteCaptureTimestamps(frame),
+ ElementsAre(kAbsoluteCaptureTimestamp));
+ }));
+ rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
+ video_header);
+}
+
+TEST_F(RtpVideoStreamReceiver2Test,
+ MissingAbsoluteCaptureTimeIsFilledWithExtrapolatedValue) {
+ constexpr uint64_t kAbsoluteCaptureTimestamp = 12;
+ constexpr int kId0 = 1;
+
+ RtpHeaderExtensionMap extension_map;
+ extension_map.Register<AbsoluteCaptureTimeExtension>(kId0);
+ RtpPacketReceived rtp_packet(&extension_map);
+ rtp_packet.SetPayloadType(kPayloadType);
+
+ rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
+ uint16_t sequence_number = 1;
+ uint32_t rtp_timestamp = 1;
+ rtp_packet.SetSequenceNumber(sequence_number);
+ rtp_packet.SetTimestamp(rtp_timestamp);
+ rtp_packet.SetSsrc(kSsrc);
+ rtp_packet.SetExtension<AbsoluteCaptureTimeExtension>(
+ AbsoluteCaptureTime{kAbsoluteCaptureTimestamp,
+ /*estimated_capture_clock_offset=*/absl::nullopt});
+
+ RTPVideoHeader video_header =
+ GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
+ data.size());
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
+ rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
+ video_header);
+
+ // Rtp packet without absolute capture time.
+ rtp_packet = RtpPacketReceived(&extension_map);
+ rtp_packet.SetPayloadType(kPayloadType);
+ rtp_packet.SetSequenceNumber(++sequence_number);
+ rtp_packet.SetTimestamp(++rtp_timestamp);
+ rtp_packet.SetSsrc(kSsrc);
+
+ // There is no absolute capture time in the second packet.
+ // Expect rtp video stream receiver to extrapolate it for the resulting video
+ // frame using absolute capture time from the previous packet.
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
+ .WillOnce(Invoke([](video_coding::EncodedFrame* frame) {
+ EXPECT_THAT(GetAbsoluteCaptureTimestamps(frame), SizeIs(1));
+ }));
+ rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
+ video_header);
+}
+
+TEST_F(RtpVideoStreamReceiver2Test,
+ NoInfiniteRecursionOnEncapsulatedRedPacket) {
+ const std::vector<uint8_t> data({
+ 0x80, // RTP version.
+ kRedPayloadType, // Payload type.
+ 0, 0, 0, 0, 0, 0, // Don't care.
+ 0, 0, 0x4, 0x57, // SSRC
+ kRedPayloadType, // RED header.
+ 0, 0, 0, 0, 0 // Don't care.
+ });
+ RtpPacketReceived packet;
+ EXPECT_TRUE(packet.Parse(data.data(), data.size()));
+ rtp_video_stream_receiver_->StartReceive();
+ rtp_video_stream_receiver_->OnRtpPacket(packet);
+}
+
+TEST_F(RtpVideoStreamReceiver2Test,
+ DropsPacketWithRedPayloadTypeAndEmptyPayload) {
+ const uint8_t kRedPayloadType = 125;
+ config_.rtp.red_payload_type = kRedPayloadType;
+ SetUp(); // re-create rtp_video_stream_receiver with red payload type.
+ // clang-format off
+ const uint8_t data[] = {
+ 0x80, // RTP version.
+ kRedPayloadType, // Payload type.
+ 0, 0, 0, 0, 0, 0, // Don't care.
+ 0, 0, 0x4, 0x57, // SSRC
+ // Empty rtp payload.
+ };
+ // clang-format on
+ RtpPacketReceived packet;
+ // Manually convert to CopyOnWriteBuffer to be sure capacity == size
+ // and asan bot can catch read buffer overflow.
+ EXPECT_TRUE(packet.Parse(rtc::CopyOnWriteBuffer(data)));
+ rtp_video_stream_receiver_->StartReceive();
+ rtp_video_stream_receiver_->OnRtpPacket(packet);
+ // Expect asan doesn't find anything.
+}
+
+TEST_F(RtpVideoStreamReceiver2Test, GenericKeyFrameBitstreamError) {
+ RtpPacketReceived rtp_packet;
+ rtp_packet.SetPayloadType(kPayloadType);
+ rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
+ rtp_packet.SetSequenceNumber(1);
+ RTPVideoHeader video_header =
+ GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
+ constexpr uint8_t expected_bitsteam[] = {1, 2, 3, 0xff};
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(
+ expected_bitsteam, sizeof(expected_bitsteam));
+ EXPECT_CALL(mock_on_complete_frame_callback_,
+ DoOnCompleteFrameFailBitstream(_));
+ rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
+ video_header);
+}
+
+class RtpVideoStreamReceiver2TestH264
+ : public RtpVideoStreamReceiver2Test,
+ public ::testing::WithParamInterface<std::string> {
+ protected:
+ RtpVideoStreamReceiver2TestH264() : RtpVideoStreamReceiver2Test(GetParam()) {}
+};
+
+INSTANTIATE_TEST_SUITE_P(SpsPpsIdrIsKeyframe,
+ RtpVideoStreamReceiver2TestH264,
+ Values("", "WebRTC-SpsPpsIdrIsH264Keyframe/Enabled/"));
+
+// Fails on MSAN: https://bugs.chromium.org/p/webrtc/issues/detail?id=11376.
+#if defined(MEMORY_SANITIZER)
+#define MAYBE_InBandSpsPps DISABLED_InBandSpsPps
+#else
+#define MAYBE_InBandSpsPps InBandSpsPps
+#endif
+TEST_P(RtpVideoStreamReceiver2TestH264, MAYBE_InBandSpsPps) {
+ rtc::CopyOnWriteBuffer sps_data;
+ RtpPacketReceived rtp_packet;
+ RTPVideoHeader sps_video_header = GetDefaultH264VideoHeader();
+ AddSps(&sps_video_header, 0, &sps_data);
+ rtp_packet.SetSequenceNumber(0);
+ rtp_packet.SetPayloadType(kPayloadType);
+ sps_video_header.is_first_packet_in_frame = true;
+ sps_video_header.frame_type = VideoFrameType::kEmptyFrame;
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(
+ kH264StartCode, sizeof(kH264StartCode));
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(sps_data.data(),
+ sps_data.size());
+ rtp_video_stream_receiver_->OnReceivedPayloadData(sps_data, rtp_packet,
+ sps_video_header);
+
+ rtc::CopyOnWriteBuffer pps_data;
+ RTPVideoHeader pps_video_header = GetDefaultH264VideoHeader();
+ AddPps(&pps_video_header, 0, 1, &pps_data);
+ rtp_packet.SetSequenceNumber(1);
+ pps_video_header.is_first_packet_in_frame = true;
+ pps_video_header.frame_type = VideoFrameType::kEmptyFrame;
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(
+ kH264StartCode, sizeof(kH264StartCode));
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(pps_data.data(),
+ pps_data.size());
+ rtp_video_stream_receiver_->OnReceivedPayloadData(pps_data, rtp_packet,
+ pps_video_header);
+
+ rtc::CopyOnWriteBuffer idr_data;
+ RTPVideoHeader idr_video_header = GetDefaultH264VideoHeader();
+ AddIdr(&idr_video_header, 1);
+ rtp_packet.SetSequenceNumber(2);
+ idr_video_header.is_first_packet_in_frame = true;
+ idr_video_header.is_last_packet_in_frame = true;
+ idr_video_header.frame_type = VideoFrameType::kVideoFrameKey;
+ const uint8_t idr[] = {0x65, 1, 2, 3};
+ idr_data.AppendData(idr);
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(
+ kH264StartCode, sizeof(kH264StartCode));
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(),
+ idr_data.size());
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
+ rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet,
+ idr_video_header);
+}
+
+TEST_P(RtpVideoStreamReceiver2TestH264, OutOfBandFmtpSpsPps) {
+ constexpr int kPayloadType = 99;
+ VideoCodec codec;
+ codec.plType = kPayloadType;
+ std::map<std::string, std::string> codec_params;
+ // Example parameter sets from https://tools.ietf.org/html/rfc3984#section-8.2
+ // .
+ codec_params.insert(
+ {cricket::kH264FmtpSpropParameterSets, "Z0IACpZTBYmI,aMljiA=="});
+ rtp_video_stream_receiver_->AddReceiveCodec(codec, codec_params,
+ /*raw_payload=*/false);
+ const uint8_t binary_sps[] = {0x67, 0x42, 0x00, 0x0a, 0x96,
+ 0x53, 0x05, 0x89, 0x88};
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(
+ kH264StartCode, sizeof(kH264StartCode));
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(binary_sps,
+ sizeof(binary_sps));
+ const uint8_t binary_pps[] = {0x68, 0xc9, 0x63, 0x88};
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(
+ kH264StartCode, sizeof(kH264StartCode));
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(binary_pps,
+ sizeof(binary_pps));
+
+ RtpPacketReceived rtp_packet;
+ RTPVideoHeader video_header = GetDefaultH264VideoHeader();
+ AddIdr(&video_header, 0);
+ rtp_packet.SetPayloadType(kPayloadType);
+ rtp_packet.SetSequenceNumber(2);
+ video_header.is_first_packet_in_frame = true;
+ video_header.is_last_packet_in_frame = true;
+ video_header.codec = kVideoCodecH264;
+ video_header.frame_type = VideoFrameType::kVideoFrameKey;
+ rtc::CopyOnWriteBuffer data({1, 2, 3});
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(
+ kH264StartCode, sizeof(kH264StartCode));
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
+ data.size());
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
+ rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
+ video_header);
+}
+
+TEST_F(RtpVideoStreamReceiver2Test, PaddingInMediaStream) {
+ RtpPacketReceived rtp_packet;
+ RTPVideoHeader video_header = GetDefaultH264VideoHeader();
+ rtc::CopyOnWriteBuffer data({1, 2, 3});
+ rtp_packet.SetPayloadType(kPayloadType);
+ rtp_packet.SetSequenceNumber(2);
+ video_header.is_first_packet_in_frame = true;
+ video_header.is_last_packet_in_frame = true;
+ video_header.codec = kVideoCodecGeneric;
+ video_header.frame_type = VideoFrameType::kVideoFrameKey;
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
+ data.size());
+
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
+ rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
+ video_header);
+
+ rtp_packet.SetSequenceNumber(3);
+ rtp_video_stream_receiver_->OnReceivedPayloadData({}, rtp_packet,
+ video_header);
+
+ rtp_packet.SetSequenceNumber(4);
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
+ video_header.frame_type = VideoFrameType::kVideoFrameDelta;
+ rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
+ video_header);
+
+ rtp_packet.SetSequenceNumber(6);
+ rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
+ video_header);
+
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
+ rtp_packet.SetSequenceNumber(5);
+ rtp_video_stream_receiver_->OnReceivedPayloadData({}, rtp_packet,
+ video_header);
+}
+
+TEST_F(RtpVideoStreamReceiver2Test, RequestKeyframeIfFirstFrameIsDelta) {
+ RtpPacketReceived rtp_packet;
+ rtp_packet.SetPayloadType(kPayloadType);
+ rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
+ rtp_packet.SetSequenceNumber(1);
+ RTPVideoHeader video_header =
+ GetGenericVideoHeader(VideoFrameType::kVideoFrameDelta);
+ EXPECT_CALL(mock_key_frame_request_sender_, RequestKeyFrame());
+ rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
+ video_header);
+}
+
+TEST_F(RtpVideoStreamReceiver2Test, RequestKeyframeWhenPacketBufferGetsFull) {
+ constexpr int kPacketBufferMaxSize = 2048;
+
+ RtpPacketReceived rtp_packet;
+ rtp_packet.SetPayloadType(kPayloadType);
+ rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
+ RTPVideoHeader video_header =
+ GetGenericVideoHeader(VideoFrameType::kVideoFrameDelta);
+ // Incomplete frames so that the packet buffer is filling up.
+ video_header.is_last_packet_in_frame = false;
+ uint16_t start_sequence_number = 1234;
+ rtp_packet.SetSequenceNumber(start_sequence_number);
+ while (rtp_packet.SequenceNumber() - start_sequence_number <
+ kPacketBufferMaxSize) {
+ rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
+ video_header);
+ rtp_packet.SetSequenceNumber(rtp_packet.SequenceNumber() + 2);
+ }
+
+ EXPECT_CALL(mock_key_frame_request_sender_, RequestKeyFrame());
+ rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
+ video_header);
+}
+
+TEST_F(RtpVideoStreamReceiver2Test, SecondarySinksGetRtpNotifications) {
+ rtp_video_stream_receiver_->StartReceive();
+
+ MockRtpPacketSink secondary_sink_1;
+ MockRtpPacketSink secondary_sink_2;
+
+ rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink_1);
+ rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink_2);
+
+ auto rtp_packet = CreateRtpPacketReceived();
+ EXPECT_CALL(secondary_sink_1, OnRtpPacket(SamePacketAs(*rtp_packet)));
+ EXPECT_CALL(secondary_sink_2, OnRtpPacket(SamePacketAs(*rtp_packet)));
+
+ rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet);
+
+ // Test tear-down.
+ rtp_video_stream_receiver_->StopReceive();
+ rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink_1);
+ rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink_2);
+}
+
+TEST_F(RtpVideoStreamReceiver2Test,
+ RemovedSecondarySinksGetNoRtpNotifications) {
+ rtp_video_stream_receiver_->StartReceive();
+
+ MockRtpPacketSink secondary_sink;
+
+ rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink);
+ rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink);
+
+ auto rtp_packet = CreateRtpPacketReceived();
+
+ EXPECT_CALL(secondary_sink, OnRtpPacket(_)).Times(0);
+
+ rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet);
+
+ // Test tear-down.
+ rtp_video_stream_receiver_->StopReceive();
+}
+
+TEST_F(RtpVideoStreamReceiver2Test,
+ OnlyRemovedSecondarySinksExcludedFromNotifications) {
+ rtp_video_stream_receiver_->StartReceive();
+
+ MockRtpPacketSink kept_secondary_sink;
+ MockRtpPacketSink removed_secondary_sink;
+
+ rtp_video_stream_receiver_->AddSecondarySink(&kept_secondary_sink);
+ rtp_video_stream_receiver_->AddSecondarySink(&removed_secondary_sink);
+ rtp_video_stream_receiver_->RemoveSecondarySink(&removed_secondary_sink);
+
+ auto rtp_packet = CreateRtpPacketReceived();
+ EXPECT_CALL(kept_secondary_sink, OnRtpPacket(SamePacketAs(*rtp_packet)));
+
+ rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet);
+
+ // Test tear-down.
+ rtp_video_stream_receiver_->StopReceive();
+ rtp_video_stream_receiver_->RemoveSecondarySink(&kept_secondary_sink);
+}
+
+TEST_F(RtpVideoStreamReceiver2Test,
+ SecondariesOfNonStartedStreamGetNoNotifications) {
+ // Explicitly showing that the stream is not in the |started| state,
+ // regardless of whether streams start out |started| or |stopped|.
+ rtp_video_stream_receiver_->StopReceive();
+
+ MockRtpPacketSink secondary_sink;
+ rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink);
+
+ auto rtp_packet = CreateRtpPacketReceived();
+ EXPECT_CALL(secondary_sink, OnRtpPacket(_)).Times(0);
+
+ rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet);
+
+ // Test tear-down.
+ rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink);
+}
+
+TEST_F(RtpVideoStreamReceiver2Test, ParseGenericDescriptorOnePacket) {
+ const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
+ const int kSpatialIndex = 1;
+
+ rtp_video_stream_receiver_->StartReceive();
+
+ RtpHeaderExtensionMap extension_map;
+ extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
+ RtpPacketReceived rtp_packet(&extension_map);
+ rtp_packet.SetPayloadType(kPayloadType);
+
+ RtpGenericFrameDescriptor generic_descriptor;
+ generic_descriptor.SetFirstPacketInSubFrame(true);
+ generic_descriptor.SetLastPacketInSubFrame(true);
+ generic_descriptor.SetFrameId(100);
+ generic_descriptor.SetSpatialLayersBitmask(1 << kSpatialIndex);
+ generic_descriptor.AddFrameDependencyDiff(90);
+ generic_descriptor.AddFrameDependencyDiff(80);
+ ASSERT_TRUE(rtp_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
+ generic_descriptor));
+
+ uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
+ memcpy(payload, data.data(), data.size());
+ // The first byte is the header, so we ignore the first byte of |data|.
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1,
+ data.size() - 1);
+
+ rtp_packet.SetMarker(true);
+ rtp_packet.SetPayloadType(kPayloadType);
+ rtp_packet.SetSequenceNumber(1);
+
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
+ .WillOnce(Invoke([kSpatialIndex](video_coding::EncodedFrame* frame) {
+ EXPECT_EQ(frame->num_references, 2U);
+ EXPECT_EQ(frame->references[0], frame->id.picture_id - 90);
+ EXPECT_EQ(frame->references[1], frame->id.picture_id - 80);
+ EXPECT_EQ(frame->id.spatial_layer, kSpatialIndex);
+ EXPECT_THAT(frame->PacketInfos(), SizeIs(1));
+ }));
+
+ rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
+}
+
+TEST_F(RtpVideoStreamReceiver2Test, ParseGenericDescriptorTwoPackets) {
+ const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
+ const int kSpatialIndex = 1;
+
+ rtp_video_stream_receiver_->StartReceive();
+
+ RtpHeaderExtensionMap extension_map;
+ extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
+ RtpPacketReceived first_packet(&extension_map);
+
+ RtpGenericFrameDescriptor first_packet_descriptor;
+ first_packet_descriptor.SetFirstPacketInSubFrame(true);
+ first_packet_descriptor.SetLastPacketInSubFrame(false);
+ first_packet_descriptor.SetFrameId(100);
+ first_packet_descriptor.SetSpatialLayersBitmask(1 << kSpatialIndex);
+ first_packet_descriptor.SetResolution(480, 360);
+ ASSERT_TRUE(first_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
+ first_packet_descriptor));
+
+ uint8_t* first_packet_payload = first_packet.SetPayloadSize(data.size());
+ memcpy(first_packet_payload, data.data(), data.size());
+ // The first byte is the header, so we ignore the first byte of |data|.
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1,
+ data.size() - 1);
+
+ first_packet.SetPayloadType(kPayloadType);
+ first_packet.SetSequenceNumber(1);
+ rtp_video_stream_receiver_->OnRtpPacket(first_packet);
+
+ RtpPacketReceived second_packet(&extension_map);
+ RtpGenericFrameDescriptor second_packet_descriptor;
+ second_packet_descriptor.SetFirstPacketInSubFrame(false);
+ second_packet_descriptor.SetLastPacketInSubFrame(true);
+ ASSERT_TRUE(second_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
+ second_packet_descriptor));
+
+ second_packet.SetMarker(true);
+ second_packet.SetPayloadType(kPayloadType);
+ second_packet.SetSequenceNumber(2);
+
+ uint8_t* second_packet_payload = second_packet.SetPayloadSize(data.size());
+ memcpy(second_packet_payload, data.data(), data.size());
+ // The first byte is the header, so we ignore the first byte of |data|.
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1,
+ data.size() - 1);
+
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
+ .WillOnce(Invoke([kSpatialIndex](video_coding::EncodedFrame* frame) {
+ EXPECT_EQ(frame->num_references, 0U);
+ EXPECT_EQ(frame->id.spatial_layer, kSpatialIndex);
+ EXPECT_EQ(frame->EncodedImage()._encodedWidth, 480u);
+ EXPECT_EQ(frame->EncodedImage()._encodedHeight, 360u);
+ EXPECT_THAT(frame->PacketInfos(), SizeIs(2));
+ }));
+
+ rtp_video_stream_receiver_->OnRtpPacket(second_packet);
+}
+
+TEST_F(RtpVideoStreamReceiver2Test, ParseGenericDescriptorRawPayload) {
+ const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
+ const int kRawPayloadType = 123;
+
+ VideoCodec codec;
+ codec.plType = kRawPayloadType;
+ rtp_video_stream_receiver_->AddReceiveCodec(codec, {}, /*raw_payload=*/true);
+ rtp_video_stream_receiver_->StartReceive();
+
+ RtpHeaderExtensionMap extension_map;
+ extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
+ RtpPacketReceived rtp_packet(&extension_map);
+
+ RtpGenericFrameDescriptor generic_descriptor;
+ generic_descriptor.SetFirstPacketInSubFrame(true);
+ generic_descriptor.SetLastPacketInSubFrame(true);
+ ASSERT_TRUE(rtp_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
+ generic_descriptor));
+
+ uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
+ memcpy(payload, data.data(), data.size());
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
+ data.size());
+
+ rtp_packet.SetMarker(true);
+ rtp_packet.SetPayloadType(kRawPayloadType);
+ rtp_packet.SetSequenceNumber(1);
+
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame);
+ rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
+}
+
+TEST_F(RtpVideoStreamReceiver2Test, UnwrapsFrameId) {
+ const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
+ const int kPayloadType = 123;
+
+ VideoCodec codec;
+ codec.plType = kPayloadType;
+ rtp_video_stream_receiver_->AddReceiveCodec(codec, {}, /*raw_payload=*/true);
+ rtp_video_stream_receiver_->StartReceive();
+ RtpHeaderExtensionMap extension_map;
+ extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
+
+ uint16_t rtp_sequence_number = 1;
+ auto inject_packet = [&](uint16_t wrapped_frame_id) {
+ RtpPacketReceived rtp_packet(&extension_map);
+
+ RtpGenericFrameDescriptor generic_descriptor;
+ generic_descriptor.SetFirstPacketInSubFrame(true);
+ generic_descriptor.SetLastPacketInSubFrame(true);
+ generic_descriptor.SetFrameId(wrapped_frame_id);
+ ASSERT_TRUE(rtp_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
+ generic_descriptor));
+
+ uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
+ ASSERT_TRUE(payload);
+ memcpy(payload, data.data(), data.size());
+ mock_on_complete_frame_callback_.ClearExpectedBitstream();
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
+ data.size());
+ rtp_packet.SetMarker(true);
+ rtp_packet.SetPayloadType(kPayloadType);
+ rtp_packet.SetSequenceNumber(++rtp_sequence_number);
+ rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
+ };
+
+ int64_t first_picture_id;
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
+ .WillOnce([&](video_coding::EncodedFrame* frame) {
+ first_picture_id = frame->id.picture_id;
+ });
+ inject_packet(/*wrapped_frame_id=*/0xffff);
+
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
+ .WillOnce([&](video_coding::EncodedFrame* frame) {
+ EXPECT_EQ(frame->id.picture_id - first_picture_id, 3);
+ });
+ inject_packet(/*wrapped_frame_id=*/0x0002);
+}
+
+class RtpVideoStreamReceiver2DependencyDescriptorTest
+ : public RtpVideoStreamReceiver2Test {
+ public:
+ RtpVideoStreamReceiver2DependencyDescriptorTest() {
+ VideoCodec codec;
+ codec.plType = payload_type_;
+ rtp_video_stream_receiver_->AddReceiveCodec(codec, {},
+ /*raw_payload=*/true);
+ extension_map_.Register<RtpDependencyDescriptorExtension>(7);
+ rtp_video_stream_receiver_->StartReceive();
+ }
+
+ // Returns some valid structure for the DependencyDescriptors.
+ // First template of that structure always fit for a key frame.
+ static FrameDependencyStructure CreateStreamStructure() {
+ FrameDependencyStructure stream_structure;
+ stream_structure.num_decode_targets = 1;
+ stream_structure.templates = {
+ GenericFrameInfo::Builder().Dtis("S").Build(),
+ GenericFrameInfo::Builder().Dtis("S").Fdiffs({1}).Build(),
+ };
+ return stream_structure;
+ }
+
+ void InjectPacketWith(const FrameDependencyStructure& stream_structure,
+ const DependencyDescriptor& dependency_descriptor) {
+ const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
+ RtpPacketReceived rtp_packet(&extension_map_);
+ ASSERT_TRUE(rtp_packet.SetExtension<RtpDependencyDescriptorExtension>(
+ stream_structure, dependency_descriptor));
+ uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
+ ASSERT_TRUE(payload);
+ memcpy(payload, data.data(), data.size());
+ mock_on_complete_frame_callback_.ClearExpectedBitstream();
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
+ data.size());
+ rtp_packet.SetMarker(true);
+ rtp_packet.SetPayloadType(payload_type_);
+ rtp_packet.SetSequenceNumber(++rtp_sequence_number_);
+ rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
+ }
+
+ private:
+ const int payload_type_ = 123;
+ RtpHeaderExtensionMap extension_map_;
+ uint16_t rtp_sequence_number_ = 321;
+};
+
+TEST_F(RtpVideoStreamReceiver2DependencyDescriptorTest, UnwrapsFrameId) {
+ FrameDependencyStructure stream_structure = CreateStreamStructure();
+
+ DependencyDescriptor keyframe_descriptor;
+ keyframe_descriptor.attached_structure =
+ std::make_unique<FrameDependencyStructure>(stream_structure);
+ keyframe_descriptor.frame_dependencies = stream_structure.templates[0];
+ keyframe_descriptor.frame_number = 0xfff0;
+ // DependencyDescriptor doesn't support reordering delta frame before
+ // keyframe. Thus feed a key frame first, then test reodered delta frames.
+ int64_t first_picture_id;
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
+ .WillOnce([&](video_coding::EncodedFrame* frame) {
+ first_picture_id = frame->id.picture_id;
+ });
+ InjectPacketWith(stream_structure, keyframe_descriptor);
+
+ DependencyDescriptor deltaframe1_descriptor;
+ deltaframe1_descriptor.frame_dependencies = stream_structure.templates[1];
+ deltaframe1_descriptor.frame_number = 0xfffe;
+
+ DependencyDescriptor deltaframe2_descriptor;
+ deltaframe1_descriptor.frame_dependencies = stream_structure.templates[1];
+ deltaframe2_descriptor.frame_number = 0x0002;
+
+ // Parser should unwrap frame ids correctly even if packets were reordered by
+ // the network.
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
+ .WillOnce([&](video_coding::EncodedFrame* frame) {
+ // 0x0002 - 0xfff0
+ EXPECT_EQ(frame->id.picture_id - first_picture_id, 18);
+ })
+ .WillOnce([&](video_coding::EncodedFrame* frame) {
+ // 0xfffe - 0xfff0
+ EXPECT_EQ(frame->id.picture_id - first_picture_id, 14);
+ });
+ InjectPacketWith(stream_structure, deltaframe2_descriptor);
+ InjectPacketWith(stream_structure, deltaframe1_descriptor);
+}
+
+TEST_F(RtpVideoStreamReceiver2DependencyDescriptorTest,
+ DropsLateDeltaFramePacketWithDependencyDescriptorExtension) {
+ FrameDependencyStructure stream_structure1 = CreateStreamStructure();
+ FrameDependencyStructure stream_structure2 = CreateStreamStructure();
+ // Make sure template ids for these two structures do not collide:
+ // adjust structure_id (that is also used as template id offset).
+ stream_structure1.structure_id = 13;
+ stream_structure2.structure_id =
+ stream_structure1.structure_id + stream_structure1.templates.size();
+
+ DependencyDescriptor keyframe1_descriptor;
+ keyframe1_descriptor.attached_structure =
+ std::make_unique<FrameDependencyStructure>(stream_structure1);
+ keyframe1_descriptor.frame_dependencies = stream_structure1.templates[0];
+ keyframe1_descriptor.frame_number = 1;
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame);
+ InjectPacketWith(stream_structure1, keyframe1_descriptor);
+
+ // Pass in 2nd key frame with different structure.
+ DependencyDescriptor keyframe2_descriptor;
+ keyframe2_descriptor.attached_structure =
+ std::make_unique<FrameDependencyStructure>(stream_structure2);
+ keyframe2_descriptor.frame_dependencies = stream_structure2.templates[0];
+ keyframe2_descriptor.frame_number = 3;
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame);
+ InjectPacketWith(stream_structure2, keyframe2_descriptor);
+
+ // Pass in late delta frame that uses structure of the 1st key frame.
+ DependencyDescriptor deltaframe_descriptor;
+ deltaframe_descriptor.frame_dependencies = stream_structure1.templates[0];
+ deltaframe_descriptor.frame_number = 2;
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame).Times(0);
+ InjectPacketWith(stream_structure1, deltaframe_descriptor);
+}
+
+TEST_F(RtpVideoStreamReceiver2DependencyDescriptorTest,
+ DropsLateKeyFramePacketWithDependencyDescriptorExtension) {
+ FrameDependencyStructure stream_structure1 = CreateStreamStructure();
+ FrameDependencyStructure stream_structure2 = CreateStreamStructure();
+ // Make sure template ids for these two structures do not collide:
+ // adjust structure_id (that is also used as template id offset).
+ stream_structure1.structure_id = 13;
+ stream_structure2.structure_id =
+ stream_structure1.structure_id + stream_structure1.templates.size();
+
+ DependencyDescriptor keyframe1_descriptor;
+ keyframe1_descriptor.attached_structure =
+ std::make_unique<FrameDependencyStructure>(stream_structure1);
+ keyframe1_descriptor.frame_dependencies = stream_structure1.templates[0];
+ keyframe1_descriptor.frame_number = 1;
+
+ DependencyDescriptor keyframe2_descriptor;
+ keyframe2_descriptor.attached_structure =
+ std::make_unique<FrameDependencyStructure>(stream_structure2);
+ keyframe2_descriptor.frame_dependencies = stream_structure2.templates[0];
+ keyframe2_descriptor.frame_number = 3;
+
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
+ .WillOnce([&](video_coding::EncodedFrame* frame) {
+ EXPECT_EQ(frame->id.picture_id & 0xFFFF, 3);
+ });
+ InjectPacketWith(stream_structure2, keyframe2_descriptor);
+ InjectPacketWith(stream_structure1, keyframe1_descriptor);
+
+ // Pass in delta frame that uses structure of the 2nd key frame. Late key
+ // frame shouldn't block it.
+ DependencyDescriptor deltaframe_descriptor;
+ deltaframe_descriptor.frame_dependencies = stream_structure2.templates[0];
+ deltaframe_descriptor.frame_number = 4;
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
+ .WillOnce([&](video_coding::EncodedFrame* frame) {
+ EXPECT_EQ(frame->id.picture_id & 0xFFFF, 4);
+ });
+ InjectPacketWith(stream_structure2, deltaframe_descriptor);
+}
+
+#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)
+TEST_F(RtpVideoStreamReceiver2Test, RepeatedSecondarySinkDisallowed) {
+ MockRtpPacketSink secondary_sink;
+
+ rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink);
+ EXPECT_DEATH(rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink),
+ "");
+
+ // Test tear-down.
+ rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink);
+}
+#endif
+
+TEST_F(RtpVideoStreamReceiver2Test, TransformFrame) {
+ rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
+ new rtc::RefCountedObject<testing::NiceMock<MockFrameTransformer>>();
+ EXPECT_CALL(*mock_frame_transformer,
+ RegisterTransformedFrameSinkCallback(_, config_.rtp.remote_ssrc));
+ auto receiver = std::make_unique<RtpVideoStreamReceiver2>(
+ Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_,
+ rtp_receive_statistics_.get(), nullptr, nullptr, process_thread_.get(),
+ &mock_nack_sender_, nullptr, &mock_on_complete_frame_callback_, nullptr,
+ mock_frame_transformer);
+ VideoCodec video_codec;
+ video_codec.plType = kPayloadType;
+ video_codec.codecType = kVideoCodecGeneric;
+ receiver->AddReceiveCodec(video_codec, {}, /*raw_payload=*/false);
+
+ RtpPacketReceived rtp_packet;
+ rtp_packet.SetPayloadType(kPayloadType);
+ rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
+ rtp_packet.SetSequenceNumber(1);
+ RTPVideoHeader video_header =
+ GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
+ data.size());
+ EXPECT_CALL(*mock_frame_transformer, Transform(_));
+ receiver->OnReceivedPayloadData(data, rtp_packet, video_header);
+
+ EXPECT_CALL(*mock_frame_transformer,
+ UnregisterTransformedFrameSinkCallback(config_.rtp.remote_ssrc));
+ receiver = nullptr;
+}
+
+// Test default behavior and when playout delay is overridden by field trial.
+const PlayoutDelay kTransmittedPlayoutDelay = {100, 200};
+const PlayoutDelay kForcedPlayoutDelay = {70, 90};
+struct PlayoutDelayOptions {
+ std::string field_trial;
+ PlayoutDelay expected_delay;
+};
+const PlayoutDelayOptions kDefaultBehavior = {
+ /*field_trial=*/"", /*expected_delay=*/kTransmittedPlayoutDelay};
+const PlayoutDelayOptions kOverridePlayoutDelay = {
+ /*field_trial=*/"WebRTC-ForcePlayoutDelay/min_ms:70,max_ms:90/",
+ /*expected_delay=*/kForcedPlayoutDelay};
+
+class RtpVideoStreamReceiver2TestPlayoutDelay
+ : public RtpVideoStreamReceiver2Test,
+ public ::testing::WithParamInterface<PlayoutDelayOptions> {
+ protected:
+ RtpVideoStreamReceiver2TestPlayoutDelay()
+ : RtpVideoStreamReceiver2Test(GetParam().field_trial) {}
+};
+
+INSTANTIATE_TEST_SUITE_P(PlayoutDelay,
+ RtpVideoStreamReceiver2TestPlayoutDelay,
+ Values(kDefaultBehavior, kOverridePlayoutDelay));
+
+TEST_P(RtpVideoStreamReceiver2TestPlayoutDelay, PlayoutDelay) {
+ rtc::CopyOnWriteBuffer payload_data({1, 2, 3, 4});
+ RtpHeaderExtensionMap extension_map;
+ extension_map.Register<PlayoutDelayLimits>(1);
+ RtpPacketToSend packet_to_send(&extension_map);
+ packet_to_send.SetPayloadType(kPayloadType);
+ packet_to_send.SetSequenceNumber(1);
+
+ // Set playout delay on outgoing packet.
+ EXPECT_TRUE(packet_to_send.SetExtension<PlayoutDelayLimits>(
+ kTransmittedPlayoutDelay));
+ uint8_t* payload = packet_to_send.AllocatePayload(payload_data.size());
+ memcpy(payload, payload_data.data(), payload_data.size());
+
+ RtpPacketReceived received_packet(&extension_map);
+ received_packet.Parse(packet_to_send.data(), packet_to_send.size());
+
+ RTPVideoHeader video_header =
+ GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
+ mock_on_complete_frame_callback_.AppendExpectedBitstream(payload_data.data(),
+ payload_data.size());
+ // Expect the playout delay of encoded frame to be the same as the transmitted
+ // playout delay unless it was overridden by a field trial.
+ EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
+ .WillOnce(Invoke([expected_playout_delay = GetParam().expected_delay](
+ video_coding::EncodedFrame* frame) {
+ EXPECT_EQ(frame->EncodedImage().playout_delay_, expected_playout_delay);
+ }));
+ rtp_video_stream_receiver_->OnReceivedPayloadData(
+ received_packet.PayloadBuffer(), received_packet, video_header);
+}
+
+} // namespace webrtc
diff --git a/video/rtp_video_stream_receiver_frame_transformer_delegate.cc b/video/rtp_video_stream_receiver_frame_transformer_delegate.cc
index 753dbca..31eb344 100644
--- a/video/rtp_video_stream_receiver_frame_transformer_delegate.cc
+++ b/video/rtp_video_stream_receiver_frame_transformer_delegate.cc
@@ -17,7 +17,6 @@
#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
#include "rtc_base/task_utils/to_queued_task.h"
#include "rtc_base/thread.h"
-#include "video/rtp_video_stream_receiver.h"
namespace webrtc {
@@ -69,7 +68,7 @@
RtpVideoStreamReceiverFrameTransformerDelegate::
RtpVideoStreamReceiverFrameTransformerDelegate(
- RtpVideoStreamReceiver* receiver,
+ RtpVideoFrameReceiver* receiver,
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
rtc::Thread* network_thread,
uint32_t ssrc)
diff --git a/video/rtp_video_stream_receiver_frame_transformer_delegate.h b/video/rtp_video_stream_receiver_frame_transformer_delegate.h
index 0a106c9..e687e7f 100644
--- a/video/rtp_video_stream_receiver_frame_transformer_delegate.h
+++ b/video/rtp_video_stream_receiver_frame_transformer_delegate.h
@@ -20,7 +20,16 @@
namespace webrtc {
-class RtpVideoStreamReceiver;
+// Called back by RtpVideoStreamReceiverFrameTransformerDelegate on the network
+// thread after transformation.
+class RtpVideoFrameReceiver {
+ public:
+ virtual void ManageFrame(
+ std::unique_ptr<video_coding::RtpFrameObject> frame) = 0;
+
+ protected:
+ virtual ~RtpVideoFrameReceiver() = default;
+};
// Delegates calls to FrameTransformerInterface to transform frames, and to
// RtpVideoStreamReceiver to manage transformed frames on the |network_thread_|.
@@ -28,7 +37,7 @@
: public TransformedFrameCallback {
public:
RtpVideoStreamReceiverFrameTransformerDelegate(
- RtpVideoStreamReceiver* receiver,
+ RtpVideoFrameReceiver* receiver,
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
rtc::Thread* network_thread,
uint32_t ssrc);
@@ -44,7 +53,7 @@
void OnTransformedFrame(
std::unique_ptr<TransformableFrameInterface> frame) override;
- // Delegates the call to RtpVideoReceiver::ManageFrame on the
+ // Delegates the call to RtpVideoFrameReceiver::ManageFrame on the
// |network_thread_|.
void ManageFrame(std::unique_ptr<TransformableFrameInterface> frame);
@@ -53,7 +62,7 @@
private:
SequenceChecker network_sequence_checker_;
- RtpVideoStreamReceiver* receiver_ RTC_GUARDED_BY(network_sequence_checker_);
+ RtpVideoFrameReceiver* receiver_ RTC_GUARDED_BY(network_sequence_checker_);
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_
RTC_GUARDED_BY(network_sequence_checker_);
rtc::Thread* const network_thread_;
diff --git a/video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc b/video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc
index b427137..a411ca6 100644
--- a/video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc
+++ b/video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc
@@ -21,11 +21,11 @@
#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
#include "modules/utility/include/process_thread.h"
#include "rtc_base/event.h"
+#include "rtc_base/ref_counted_object.h"
#include "rtc_base/task_utils/to_queued_task.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "test/mock_frame_transformer.h"
-#include "video/rtp_video_stream_receiver.h"
namespace webrtc {
namespace {
@@ -47,70 +47,10 @@
return CreateRtpFrameObject(RTPVideoHeader());
}
-class FakeTransport : public Transport {
+class TestRtpVideoFrameReceiver : public RtpVideoFrameReceiver {
public:
- bool SendRtp(const uint8_t* packet,
- size_t length,
- const PacketOptions& options) {
- return true;
- }
- bool SendRtcp(const uint8_t* packet, size_t length) { return true; }
-};
-
-class FakeNackSender : public NackSender {
- public:
- void SendNack(const std::vector<uint16_t>& sequence_numbers) {}
- void SendNack(const std::vector<uint16_t>& sequence_numbers,
- bool buffering_allowed) {}
-};
-
-class FakeOnCompleteFrameCallback
- : public video_coding::OnCompleteFrameCallback {
- public:
- void OnCompleteFrame(
- std::unique_ptr<video_coding::EncodedFrame> frame) override {}
-};
-
-class TestRtpVideoStreamReceiverInitializer {
- public:
- TestRtpVideoStreamReceiverInitializer()
- : test_config_(nullptr),
- test_process_thread_(ProcessThread::Create("TestThread")) {
- test_config_.rtp.remote_ssrc = 1111;
- test_config_.rtp.local_ssrc = 2222;
- test_rtp_receive_statistics_ =
- ReceiveStatistics::Create(Clock::GetRealTimeClock());
- }
-
- protected:
- VideoReceiveStream::Config test_config_;
- FakeTransport fake_transport_;
- FakeNackSender fake_nack_sender_;
- FakeOnCompleteFrameCallback fake_on_complete_frame_callback_;
- std::unique_ptr<ProcessThread> test_process_thread_;
- std::unique_ptr<ReceiveStatistics> test_rtp_receive_statistics_;
-};
-
-class TestRtpVideoStreamReceiver : public TestRtpVideoStreamReceiverInitializer,
- public RtpVideoStreamReceiver {
- public:
- TestRtpVideoStreamReceiver()
- : TestRtpVideoStreamReceiverInitializer(),
- RtpVideoStreamReceiver(Clock::GetRealTimeClock(),
- &fake_transport_,
- nullptr,
- nullptr,
- &test_config_,
- test_rtp_receive_statistics_.get(),
- nullptr,
- nullptr,
- test_process_thread_.get(),
- &fake_nack_sender_,
- nullptr,
- &fake_on_complete_frame_callback_,
- nullptr,
- nullptr) {}
- ~TestRtpVideoStreamReceiver() override = default;
+ TestRtpVideoFrameReceiver() {}
+ ~TestRtpVideoFrameReceiver() override = default;
MOCK_METHOD(void,
ManageFrame,
@@ -120,7 +60,7 @@
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
RegisterTransformedFrameCallbackSinkOnInit) {
- TestRtpVideoStreamReceiver receiver;
+ TestRtpVideoFrameReceiver receiver;
rtc::scoped_refptr<MockFrameTransformer> frame_transformer(
new rtc::RefCountedObject<MockFrameTransformer>());
rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> delegate(
@@ -134,7 +74,7 @@
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
UnregisterTransformedFrameSinkCallbackOnReset) {
- TestRtpVideoStreamReceiver receiver;
+ TestRtpVideoFrameReceiver receiver;
rtc::scoped_refptr<MockFrameTransformer> frame_transformer(
new rtc::RefCountedObject<MockFrameTransformer>());
rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> delegate(
@@ -146,7 +86,7 @@
}
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, TransformFrame) {
- TestRtpVideoStreamReceiver receiver;
+ TestRtpVideoFrameReceiver receiver;
rtc::scoped_refptr<MockFrameTransformer> frame_transformer(
new rtc::RefCountedObject<testing::NiceMock<MockFrameTransformer>>());
rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> delegate(
@@ -160,7 +100,7 @@
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
ManageFrameOnTransformedFrame) {
- TestRtpVideoStreamReceiver receiver;
+ TestRtpVideoFrameReceiver receiver;
rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer(
new rtc::RefCountedObject<NiceMock<MockFrameTransformer>>());
rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> delegate =
@@ -186,7 +126,7 @@
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
TransformableFrameMetadataHasCorrectValue) {
- TestRtpVideoStreamReceiver receiver;
+ TestRtpVideoFrameReceiver receiver;
rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
new rtc::RefCountedObject<NiceMock<MockFrameTransformer>>();
rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> delegate =
diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc
index b1b482d..1470123 100644
--- a/video/video_receive_stream2.cc
+++ b/video/video_receive_stream2.cc
@@ -49,7 +49,7 @@
#include "system_wrappers/include/field_trial.h"
#include "video/call_stats2.h"
#include "video/frame_dumping_decoder.h"
-#include "video/receive_statistics_proxy.h"
+#include "video/receive_statistics_proxy2.h"
namespace webrtc {
diff --git a/video/video_receive_stream2.h b/video/video_receive_stream2.h
index f8cd65d..0eab5dd 100644
--- a/video/video_receive_stream2.h
+++ b/video/video_receive_stream2.h
@@ -30,7 +30,7 @@
#include "system_wrappers/include/clock.h"
#include "video/receive_statistics_proxy2.h"
#include "video/rtp_streams_synchronizer2.h"
-#include "video/rtp_video_stream_receiver.h"
+#include "video/rtp_video_stream_receiver2.h"
#include "video/transport_adapter.h"
#include "video/video_stream_decoder2.h"
@@ -199,7 +199,7 @@
std::unique_ptr<VCMTiming> timing_; // Jitter buffer experiment.
VideoReceiver2 video_receiver_;
std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> incoming_video_stream_;
- RtpVideoStreamReceiver rtp_video_stream_receiver_;
+ RtpVideoStreamReceiver2 rtp_video_stream_receiver_;
std::unique_ptr<VideoStreamDecoder> video_stream_decoder_;
RtpStreamsSynchronizer rtp_stream_sync_;
