| /* |
| * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #include "video/video_stream_buffer_controller.h" |
| |
| #include <stdint.h> |
| |
| #include <limits> |
| #include <memory> |
| #include <optional> |
| #include <string> |
| #include <tuple> |
| #include <utility> |
| #include <vector> |
| |
| #include "absl/types/variant.h" |
| #include "api/metronome/test/fake_metronome.h" |
| #include "api/units/frequency.h" |
| #include "api/units/time_delta.h" |
| #include "api/units/timestamp.h" |
| #include "api/video/video_content_type.h" |
| #include "api/video/video_timing.h" |
| #include "rtc_base/checks.h" |
| #include "test/fake_encoded_frame.h" |
| #include "test/gmock.h" |
| #include "test/gtest.h" |
| #include "test/scoped_key_value_config.h" |
| #include "test/time_controller/simulated_time_controller.h" |
| #include "video/decode_synchronizer.h" |
| #include "video/task_queue_frame_decode_scheduler.h" |
| |
| using ::testing::_; |
| using ::testing::AllOf; |
| using ::testing::Contains; |
| using ::testing::Each; |
| using ::testing::Eq; |
| using ::testing::IsEmpty; |
| using ::testing::Matches; |
| using ::testing::Ne; |
| using ::testing::Not; |
| using ::testing::Optional; |
| using ::testing::Pointee; |
| using ::testing::SizeIs; |
| using ::testing::VariantWith; |
| |
| namespace webrtc { |
| |
| namespace { |
| |
| constexpr size_t kFrameSize = 10; |
| constexpr uint32_t kFps30Rtp = 90000 / 30; |
| constexpr TimeDelta kFps30Delay = 1 / Frequency::Hertz(30); |
| constexpr Timestamp kClockStart = Timestamp::Millis(1000); |
| |
| auto TimedOut() { |
| return Optional(VariantWith<TimeDelta>(_)); |
| } |
| |
| auto Frame(testing::Matcher<EncodedFrame> m) { |
| return Optional(VariantWith<std::unique_ptr<EncodedFrame>>(Pointee(m))); |
| } |
| |
| std::unique_ptr<test::FakeEncodedFrame> WithReceiveTimeFromRtpTimestamp( |
| std::unique_ptr<test::FakeEncodedFrame> frame) { |
| if (frame->RtpTimestamp() == 0) { |
| frame->SetReceivedTime(kClockStart.ms()); |
| } else { |
| frame->SetReceivedTime( |
| TimeDelta::Seconds(frame->RtpTimestamp() / 90000.0).ms() + |
| kClockStart.ms()); |
| } |
| return frame; |
| } |
| |
| class VCMTimingTest : public VCMTiming { |
| public: |
| using VCMTiming::VCMTiming; |
| void IncomingTimestamp(uint32_t rtp_timestamp, |
| Timestamp last_packet_time) override { |
| IncomingTimestampMocked(rtp_timestamp, last_packet_time); |
| VCMTiming::IncomingTimestamp(rtp_timestamp, last_packet_time); |
| } |
| |
| MOCK_METHOD(void, |
| IncomingTimestampMocked, |
| (uint32_t rtp_timestamp, Timestamp last_packet_time), |
| ()); |
| }; |
| |
| class VideoStreamBufferControllerStatsObserverMock |
| : public VideoStreamBufferControllerStatsObserver { |
| public: |
| MOCK_METHOD(void, |
| OnCompleteFrame, |
| (bool is_keyframe, |
| size_t size_bytes, |
| VideoContentType content_type), |
| (override)); |
| MOCK_METHOD(void, OnDroppedFrames, (uint32_t num_dropped), (override)); |
| MOCK_METHOD(void, |
| OnDecodableFrame, |
| (TimeDelta jitter_buffer_delay, |
| TimeDelta target_delay, |
| TimeDelta minimum_delay), |
| (override)); |
| MOCK_METHOD(void, |
| OnFrameBufferTimingsUpdated, |
| (int estimated_max_decode_time_ms, |
| int current_delay_ms, |
| int target_delay_ms, |
| int jitter_delay_ms, |
| int min_playout_delay_ms, |
| int render_delay_ms), |
| (override)); |
| MOCK_METHOD(void, |
| OnTimingFrameInfoUpdated, |
| (const TimingFrameInfo& info), |
| (override)); |
| }; |
| |
| } // namespace |
| |
| constexpr auto kMaxWaitForKeyframe = TimeDelta::Millis(500); |
| constexpr auto kMaxWaitForFrame = TimeDelta::Millis(1500); |
| class VideoStreamBufferControllerFixture |
| : public ::testing::WithParamInterface<std::tuple<bool, std::string>>, |
| public FrameSchedulingReceiver { |
| public: |
| VideoStreamBufferControllerFixture() |
| : sync_decoding_(std::get<0>(GetParam())), |
| field_trials_(std::get<1>(GetParam())), |
| time_controller_(kClockStart), |
| clock_(time_controller_.GetClock()), |
| fake_metronome_(TimeDelta::Millis(16)), |
| decode_sync_(clock_, |
| &fake_metronome_, |
| time_controller_.GetMainThread()), |
| timing_(clock_, field_trials_), |
| buffer_(std::make_unique<VideoStreamBufferController>( |
| clock_, |
| time_controller_.GetMainThread(), |
| &timing_, |
| &stats_callback_, |
| this, |
| kMaxWaitForKeyframe, |
| kMaxWaitForFrame, |
| sync_decoding_ ? decode_sync_.CreateSynchronizedFrameScheduler() |
| : std::make_unique<TaskQueueFrameDecodeScheduler>( |
| clock_, |
| time_controller_.GetMainThread()), |
| field_trials_)) { |
| // Avoid starting with negative render times. |
| timing_.set_min_playout_delay(TimeDelta::Millis(10)); |
| |
| ON_CALL(stats_callback_, OnDroppedFrames) |
| .WillByDefault( |
| [this](auto num_dropped) { dropped_frames_ += num_dropped; }); |
| } |
| |
| ~VideoStreamBufferControllerFixture() override { |
| if (buffer_) { |
| buffer_->Stop(); |
| } |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| } |
| |
| void OnEncodedFrame(std::unique_ptr<EncodedFrame> frame) override { |
| RTC_DCHECK(frame); |
| SetWaitResult(std::move(frame)); |
| } |
| |
| void OnDecodableFrameTimeout(TimeDelta wait_time) override { |
| SetWaitResult(wait_time); |
| } |
| |
| using WaitResult = |
| absl::variant<std::unique_ptr<EncodedFrame>, TimeDelta /*wait_time*/>; |
| |
| std::optional<WaitResult> WaitForFrameOrTimeout(TimeDelta wait) { |
| if (wait_result_) { |
| return std::move(wait_result_); |
| } |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| if (wait_result_) { |
| return std::move(wait_result_); |
| } |
| |
| Timestamp now = clock_->CurrentTime(); |
| // TODO(bugs.webrtc.org/13756): Remove this when rtc::Thread uses uses |
| // Timestamp instead of an integer milliseconds. This extra wait is needed |
| // for some tests that use the metronome. This is due to rounding |
| // milliseconds, affecting the precision of simulated time controller uses |
| // when posting tasks from threads. |
| TimeDelta potential_extra_wait = |
| Timestamp::Millis((now + wait).ms()) - (now + wait); |
| |
| time_controller_.AdvanceTime(wait); |
| if (potential_extra_wait > TimeDelta::Zero()) { |
| time_controller_.AdvanceTime(potential_extra_wait); |
| } |
| return std::move(wait_result_); |
| } |
| |
| void StartNextDecode() { |
| ResetLastResult(); |
| buffer_->StartNextDecode(false); |
| } |
| |
| void StartNextDecodeForceKeyframe() { |
| ResetLastResult(); |
| buffer_->StartNextDecode(true); |
| } |
| |
| void ResetLastResult() { wait_result_.reset(); } |
| |
| int dropped_frames() const { return dropped_frames_; } |
| |
| protected: |
| const bool sync_decoding_; |
| test::ScopedKeyValueConfig field_trials_; |
| GlobalSimulatedTimeController time_controller_; |
| Clock* const clock_; |
| test::FakeMetronome fake_metronome_; |
| DecodeSynchronizer decode_sync_; |
| |
| ::testing::NiceMock<VCMTimingTest> timing_; |
| ::testing::NiceMock<VideoStreamBufferControllerStatsObserverMock> |
| stats_callback_; |
| std::unique_ptr<VideoStreamBufferController> buffer_; |
| |
| private: |
| void SetWaitResult(WaitResult result) { |
| RTC_DCHECK(!wait_result_); |
| if (absl::holds_alternative<std::unique_ptr<EncodedFrame>>(result)) { |
| RTC_DCHECK(absl::get<std::unique_ptr<EncodedFrame>>(result)); |
| } |
| wait_result_.emplace(std::move(result)); |
| } |
| |
| uint32_t dropped_frames_ = 0; |
| std::optional<WaitResult> wait_result_; |
| }; |
| |
| class VideoStreamBufferControllerTest |
| : public ::testing::Test, |
| public VideoStreamBufferControllerFixture {}; |
| |
| TEST_P(VideoStreamBufferControllerTest, |
| InitialTimeoutAfterKeyframeTimeoutPeriod) { |
| StartNextDecodeForceKeyframe(); |
| // No frame inserted. Timeout expected. |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), TimedOut()); |
| |
| // No new timeout set since receiver has not started new decode. |
| ResetLastResult(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), Eq(std::nullopt)); |
| |
| // Now that receiver has asked for new frame, a new timeout can occur. |
| StartNextDecodeForceKeyframe(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), TimedOut()); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, KeyFramesAreScheduled) { |
| StartNextDecodeForceKeyframe(); |
| time_controller_.AdvanceTime(TimeDelta::Millis(50)); |
| |
| auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build(); |
| buffer_->InsertFrame(std::move(frame)); |
| |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, |
| DeltaFrameTimeoutAfterKeyframeExtracted) { |
| StartNextDecodeForceKeyframe(); |
| |
| time_controller_.AdvanceTime(TimeDelta::Millis(50)); |
| auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build(); |
| buffer_->InsertFrame(std::move(frame)); |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), |
| Frame(test::WithId(0))); |
| |
| StartNextDecode(); |
| time_controller_.AdvanceTime(TimeDelta::Millis(50)); |
| |
| // Timeouts should now happen at the normal frequency. |
| const int expected_timeouts = 5; |
| for (int i = 0; i < expected_timeouts; ++i) { |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); |
| StartNextDecode(); |
| } |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, DependantFramesAreScheduled) { |
| StartNextDecodeForceKeyframe(); |
| buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| StartNextDecode(); |
| |
| time_controller_.AdvanceTime(kFps30Delay); |
| buffer_->InsertFrame(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, SpatialLayersAreScheduled) { |
| StartNextDecodeForceKeyframe(); |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(0).SpatialLayer(0).Time(0).Build())); |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(1).SpatialLayer(1).Time(0).Build())); |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(2).SpatialLayer(2).Time(0).AsLast().Build())); |
| EXPECT_THAT( |
| WaitForFrameOrTimeout(TimeDelta::Zero()), |
| Frame(AllOf(test::WithId(0), test::FrameWithSize(3 * kFrameSize)))); |
| |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(3).Time(kFps30Rtp).SpatialLayer(0).Build())); |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(4).Time(kFps30Rtp).SpatialLayer(1).Build())); |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| .Id(5) |
| .Time(kFps30Rtp) |
| .SpatialLayer(2) |
| .AsLast() |
| .Build())); |
| |
| StartNextDecode(); |
| EXPECT_THAT( |
| WaitForFrameOrTimeout(kFps30Delay * 10), |
| Frame(AllOf(test::WithId(3), test::FrameWithSize(3 * kFrameSize)))); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, |
| OutstandingFrameTasksAreCancelledAfterDeletion) { |
| StartNextDecodeForceKeyframe(); |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build())); |
| // Get keyframe. Delta frame should now be scheduled. |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| StartNextDecode(); |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build())); |
| buffer_->Stop(); |
| // Wait for 2x max wait time. Since we stopped, this should cause no timeouts |
| // or frame-ready callbacks. |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame * 2), Eq(std::nullopt)); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, FramesWaitForDecoderToComplete) { |
| StartNextDecodeForceKeyframe(); |
| |
| // Start with a keyframe. |
| buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| ResetLastResult(); |
| // Insert a delta frame. |
| buffer_->InsertFrame(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build()); |
| |
| // Advancing time should not result in a frame since the scheduler has not |
| // been signalled that we are ready. |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Eq(std::nullopt)); |
| // Signal ready. |
| StartNextDecode(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, LateFrameDropped) { |
| StartNextDecodeForceKeyframe(); |
| // F1 |
| // / |
| // F0 --> F2 |
| buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| // Start with a keyframe. |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| StartNextDecode(); |
| |
| // Simulate late F1 which arrives after F2. |
| time_controller_.AdvanceTime(kFps30Delay * 2); |
| buffer_->InsertFrame(test::FakeFrameBuilder() |
| .Id(2) |
| .Time(2 * kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2))); |
| |
| StartNextDecode(); |
| |
| buffer_->InsertFrame(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(1 * kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build()); |
| // Confirm frame 1 is never scheduled by timing out. |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, FramesFastForwardOnSystemHalt) { |
| StartNextDecodeForceKeyframe(); |
| // F1 |
| // / |
| // F0 --> F2 |
| buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| |
| // Start with a keyframe. |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| time_controller_.AdvanceTime(kFps30Delay); |
| buffer_->InsertFrame(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build()); |
| time_controller_.AdvanceTime(kFps30Delay); |
| buffer_->InsertFrame(test::FakeFrameBuilder() |
| .Id(2) |
| .Time(2 * kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build()); |
| |
| // Halting time should result in F1 being skipped. |
| time_controller_.AdvanceTime(kFps30Delay * 2); |
| StartNextDecode(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2))); |
| EXPECT_EQ(dropped_frames(), 1); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, ForceKeyFrame) { |
| StartNextDecodeForceKeyframe(); |
| // Initial keyframe. |
| buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| StartNextDecodeForceKeyframe(); |
| |
| // F2 is the next keyframe, and should be extracted since a keyframe was |
| // forced. |
| buffer_->InsertFrame(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build()); |
| buffer_->InsertFrame( |
| test::FakeFrameBuilder().Id(2).Time(kFps30Rtp * 2).AsLast().Build()); |
| |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay * 3), Frame(test::WithId(2))); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, SlowDecoderDropsTemporalLayers) { |
| StartNextDecodeForceKeyframe(); |
| // 2 temporal layers, at 15fps per layer to make 30fps total. |
| // Decoder is slower than 30fps, so last_frame() will be skipped. |
| // F1 --> F3 --> F5 |
| // / / / |
| // F0 --> F2 --> F4 |
| buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| // Keyframe received. |
| // Don't start next decode until slow delay. |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| time_controller_.AdvanceTime(kFps30Delay); |
| buffer_->InsertFrame(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(1 * kFps30Rtp) |
| .Refs({0}) |
| .AsLast() |
| .Build()); |
| time_controller_.AdvanceTime(kFps30Delay); |
| buffer_->InsertFrame(test::FakeFrameBuilder() |
| .Id(2) |
| .Time(2 * kFps30Rtp) |
| .Refs({0}) |
| .AsLast() |
| .Build()); |
| |
| // Simulate decode taking 3x FPS rate. |
| time_controller_.AdvanceTime(kFps30Delay * 1.5); |
| StartNextDecode(); |
| // F2 is the best frame since decoding was so slow that F1 is too old. |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay * 2), Frame(test::WithId(2))); |
| EXPECT_EQ(dropped_frames(), 1); |
| time_controller_.AdvanceTime(kFps30Delay / 2); |
| |
| buffer_->InsertFrame(test::FakeFrameBuilder() |
| .Id(3) |
| .Time(3 * kFps30Rtp) |
| .Refs({1, 2}) |
| .AsLast() |
| .Build()); |
| time_controller_.AdvanceTime(kFps30Delay / 2); |
| buffer_->InsertFrame(test::FakeFrameBuilder() |
| .Id(4) |
| .Time(4 * kFps30Rtp) |
| .Refs({2}) |
| .AsLast() |
| .Build()); |
| time_controller_.AdvanceTime(kFps30Delay / 2); |
| |
| // F4 is the best frame since decoding was so slow that F1 is too old. |
| time_controller_.AdvanceTime(kFps30Delay); |
| StartNextDecode(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(4))); |
| |
| buffer_->InsertFrame(test::FakeFrameBuilder() |
| .Id(5) |
| .Time(5 * kFps30Rtp) |
| .Refs({3, 4}) |
| .AsLast() |
| .Build()); |
| time_controller_.AdvanceTime(kFps30Delay / 2); |
| |
| // F5 is not decodable since F4 was decoded, so a timeout is expected. |
| time_controller_.AdvanceTime(TimeDelta::Millis(10)); |
| StartNextDecode(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); |
| // TODO(bugs.webrtc.org/13343): This should be 2 dropped frames since frames 1 |
| // and 3 were dropped. However, frame_buffer2 does not mark frame 3 as dropped |
| // which is a bug. Uncomment below when that is fixed for frame_buffer2 is |
| // deleted. |
| // EXPECT_EQ(dropped_frames(), 2); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, |
| NewFrameInsertedWhileWaitingToReleaseFrame) { |
| StartNextDecodeForceKeyframe(); |
| // Initial keyframe. |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build())); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| time_controller_.AdvanceTime(kFps30Delay / 2); |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kFps30Rtp) |
| .Refs({0}) |
| .AsLast() |
| .Build())); |
| StartNextDecode(); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Eq(std::nullopt)); |
| |
| // Scheduler is waiting to deliver Frame 1 now. Insert Frame 2. Frame 1 should |
| // be delivered still. |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| .Id(2) |
| .Time(kFps30Rtp * 2) |
| .Refs({0}) |
| .AsLast() |
| .Build())); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, SameFrameNotScheduledTwice) { |
| // A frame could be scheduled twice if last_frame() arrive out-of-order but |
| // the older frame is old enough to be fast forwarded. |
| // |
| // 1. F2 arrives and is scheduled. |
| // 2. F3 arrives, but scheduling will not change since F2 is next. |
| // 3. F1 arrives late and scheduling is checked since it is before F2. F1 |
| // fast-forwarded since it is older. |
| // |
| // F2 is the best frame, but should only be scheduled once, followed by F3. |
| StartNextDecodeForceKeyframe(); |
| |
| // First keyframe. |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build())); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Millis(15)), |
| Frame(test::WithId(0))); |
| |
| StartNextDecode(); |
| |
| // F2 arrives and is scheduled. |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(2).Time(2 * kFps30Rtp).AsLast().Build())); |
| |
| // F3 arrives before F2 is extracted. |
| time_controller_.AdvanceTime(kFps30Delay); |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(3).Time(3 * kFps30Rtp).AsLast().Build())); |
| |
| // F1 arrives and is fast-forwarded since it is too late. |
| // F2 is already scheduled and should not be rescheduled. |
| time_controller_.AdvanceTime(kFps30Delay / 2); |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(1).Time(1 * kFps30Rtp).AsLast().Build())); |
| |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2))); |
| StartNextDecode(); |
| |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(3))); |
| StartNextDecode(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); |
| EXPECT_EQ(dropped_frames(), 1); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, TestStatsCallback) { |
| EXPECT_CALL(stats_callback_, |
| OnCompleteFrame(true, kFrameSize, VideoContentType::UNSPECIFIED)); |
| EXPECT_CALL(stats_callback_, OnDecodableFrame); |
| EXPECT_CALL(stats_callback_, OnFrameBufferTimingsUpdated); |
| |
| // Fake timing having received decoded frame. |
| timing_.StopDecodeTimer(TimeDelta::Millis(1), clock_->CurrentTime()); |
| StartNextDecodeForceKeyframe(); |
| buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| // Flush stats posted on the decode queue. |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, |
| FrameCompleteCalledOnceForDuplicateFrame) { |
| EXPECT_CALL(stats_callback_, |
| OnCompleteFrame(true, kFrameSize, VideoContentType::UNSPECIFIED)) |
| .Times(1); |
| |
| StartNextDecodeForceKeyframe(); |
| buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| // Flush stats posted on the decode queue. |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, |
| FrameCompleteCalledOnceForSingleTemporalUnit) { |
| StartNextDecodeForceKeyframe(); |
| |
| // `OnCompleteFrame` should not be called for the first two frames since they |
| // do not complete the temporal layer. |
| EXPECT_CALL(stats_callback_, OnCompleteFrame(_, _, _)).Times(0); |
| buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).Build()); |
| buffer_->InsertFrame( |
| test::FakeFrameBuilder().Id(1).Time(0).Refs({0}).Build()); |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| // Flush stats posted on the decode queue. |
| ::testing::Mock::VerifyAndClearExpectations(&stats_callback_); |
| |
| // Note that this frame is not marked as a keyframe since the last spatial |
| // layer has dependencies. |
| EXPECT_CALL(stats_callback_, |
| OnCompleteFrame(false, kFrameSize, VideoContentType::UNSPECIFIED)) |
| .Times(1); |
| buffer_->InsertFrame( |
| test::FakeFrameBuilder().Id(2).Time(0).Refs({0, 1}).AsLast().Build()); |
| // Flush stats posted on the decode queue. |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, |
| FrameCompleteCalledOnceForCompleteTemporalUnit) { |
| // FrameBuffer2 logs the complete frame on the arrival of the last layer. |
| StartNextDecodeForceKeyframe(); |
| |
| // `OnCompleteFrame` should not be called for the first two frames since they |
| // do not complete the temporal layer. Frame 1 arrives later, at which time |
| // this frame can finally be considered complete. |
| EXPECT_CALL(stats_callback_, OnCompleteFrame(_, _, _)).Times(0); |
| buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).Build()); |
| buffer_->InsertFrame( |
| test::FakeFrameBuilder().Id(2).Time(0).Refs({0, 1}).AsLast().Build()); |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| // Flush stats posted on the decode queue. |
| ::testing::Mock::VerifyAndClearExpectations(&stats_callback_); |
| |
| EXPECT_CALL(stats_callback_, |
| OnCompleteFrame(false, kFrameSize, VideoContentType::UNSPECIFIED)) |
| .Times(1); |
| buffer_->InsertFrame( |
| test::FakeFrameBuilder().Id(1).Time(0).Refs({0}).Build()); |
| // Flush stats posted on the decode queue. |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| } |
| |
| // Note: This test takes a long time to run if the fake metronome is active. |
| // Since the test needs to wait for the timestamp to rollover, it has a fake |
| // delay of around 6.5 hours. Even though time is simulated, this will be |
| // around 1,500,000 metronome tick invocations. |
| TEST_P(VideoStreamBufferControllerTest, NextFrameWithOldTimestamp) { |
| // Test inserting 31 frames and pause the stream for a long time before |
| // frame 32. |
| StartNextDecodeForceKeyframe(); |
| constexpr uint32_t kBaseRtp = std::numeric_limits<uint32_t>::max() / 2; |
| |
| // First keyframe. The receive time must be explicitly set in this test since |
| // the RTP derived time used in all tests does not work when the long pause |
| // happens later in the test. |
| buffer_->InsertFrame(test::FakeFrameBuilder() |
| .Id(0) |
| .Time(kBaseRtp) |
| .ReceivedTime(clock_->CurrentTime()) |
| .AsLast() |
| .Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(0))); |
| |
| // 1 more frame to warmup VCMTiming for 30fps. |
| StartNextDecode(); |
| buffer_->InsertFrame(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kBaseRtp + kFps30Rtp) |
| .ReceivedTime(clock_->CurrentTime()) |
| .AsLast() |
| .Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); |
| |
| // Pause the stream for such a long time it incurs an RTP timestamp rollover |
| // by over half. |
| constexpr uint32_t kLastRtp = kBaseRtp + kFps30Rtp; |
| constexpr uint32_t kRolloverRtp = |
| kLastRtp + std::numeric_limits<uint32_t>::max() / 2 + 1; |
| constexpr Frequency kRtpHz = Frequency::KiloHertz(90); |
| // Pause for corresponding delay such that RTP timestamp would increase this |
| // much at 30fps. |
| constexpr TimeDelta kRolloverDelay = |
| (std::numeric_limits<uint32_t>::max() / 2 + 1) / kRtpHz; |
| |
| // Avoid timeout being set while waiting for the frame and before the receiver |
| // is ready. |
| ResetLastResult(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), Eq(std::nullopt)); |
| time_controller_.AdvanceTime(kRolloverDelay - kMaxWaitForFrame); |
| StartNextDecode(); |
| buffer_->InsertFrame(test::FakeFrameBuilder() |
| .Id(2) |
| .Time(kRolloverRtp) |
| .ReceivedTime(clock_->CurrentTime()) |
| .AsLast() |
| .Build()); |
| // FrameBuffer2 drops the frame, while FrameBuffer3 will continue the stream. |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2))); |
| } |
| |
| TEST_P(VideoStreamBufferControllerTest, |
| FrameNotSetForDecodedIfFrameBufferBecomesNonDecodable) { |
| // This can happen if the frame buffer receives non-standard input. This test |
| // will simply clear the frame buffer to replicate this. |
| StartNextDecodeForceKeyframe(); |
| // Initial keyframe. |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(0).Time(0).SpatialLayer(1).AsLast().Build())); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| // Insert a frame that will become non-decodable. |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| .Id(11) |
| .Time(kFps30Rtp) |
| .Refs({0}) |
| .SpatialLayer(1) |
| .AsLast() |
| .Build())); |
| StartNextDecode(); |
| // Second layer inserted after last layer for the same frame out-of-order. |
| // This second frame requires some older frame to be decoded and so now the |
| // super-frame is no longer decodable despite already being scheduled. |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| .Id(10) |
| .Time(kFps30Rtp) |
| .SpatialLayer(0) |
| .Refs({2}) |
| .Build())); |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); |
| |
| // Ensure that this frame can be decoded later. |
| StartNextDecode(); |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| .Id(2) |
| .Time(kFps30Rtp / 2) |
| .SpatialLayer(0) |
| .Refs({0}) |
| .AsLast() |
| .Build())); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2))); |
| StartNextDecode(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(10))); |
| } |
| |
| INSTANTIATE_TEST_SUITE_P(VideoStreamBufferController, |
| VideoStreamBufferControllerTest, |
| ::testing::Combine(::testing::Bool(), |
| ::testing::Values("")), |
| [](const auto& info) { |
| return std::get<0>(info.param) ? "SyncDecoding" |
| : "UnsyncedDecoding"; |
| }); |
| |
| class LowLatencyVideoStreamBufferControllerTest |
| : public ::testing::Test, |
| public VideoStreamBufferControllerFixture {}; |
| |
| TEST_P(LowLatencyVideoStreamBufferControllerTest, |
| FramesDecodedInstantlyWithLowLatencyRendering) { |
| // Initial keyframe. |
| StartNextDecodeForceKeyframe(); |
| timing_.set_min_playout_delay(TimeDelta::Zero()); |
| timing_.set_max_playout_delay(TimeDelta::Millis(10)); |
| // Playout delay of 0 implies low-latency rendering. |
| auto frame = test::FakeFrameBuilder() |
| .Id(0) |
| .Time(0) |
| .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(10)}) |
| .AsLast() |
| .Build(); |
| buffer_->InsertFrame(std::move(frame)); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| // Delta frame would normally wait here, but should decode at the pacing rate |
| // in low-latency mode. |
| StartNextDecode(); |
| frame = test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kFps30Rtp) |
| .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(10)}) |
| .AsLast() |
| .Build(); |
| buffer_->InsertFrame(std::move(frame)); |
| // Pacing is set to 16ms in the field trial so we should not decode yet. |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Eq(std::nullopt)); |
| time_controller_.AdvanceTime(TimeDelta::Millis(16)); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1))); |
| } |
| |
| TEST_P(LowLatencyVideoStreamBufferControllerTest, ZeroPlayoutDelayFullQueue) { |
| // Initial keyframe. |
| StartNextDecodeForceKeyframe(); |
| timing_.set_min_playout_delay(TimeDelta::Zero()); |
| timing_.set_max_playout_delay(TimeDelta::Millis(10)); |
| auto frame = test::FakeFrameBuilder() |
| .Id(0) |
| .Time(0) |
| .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(10)}) |
| .AsLast() |
| .Build(); |
| // Playout delay of 0 implies low-latency rendering. |
| buffer_->InsertFrame(std::move(frame)); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| // Queue up 5 frames (configured max queue size for 0-playout delay pacing). |
| for (int id = 1; id <= 6; ++id) { |
| frame = test::FakeFrameBuilder() |
| .Id(id) |
| .Time(kFps30Rtp * id) |
| .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(10)}) |
| .AsLast() |
| .Build(); |
| buffer_->InsertFrame(std::move(frame)); |
| } |
| |
| // The queue is at its max size for zero playout delay pacing, so the pacing |
| // should be ignored and the next frame should be decoded instantly. |
| StartNextDecode(); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1))); |
| } |
| |
| TEST_P(LowLatencyVideoStreamBufferControllerTest, |
| MinMaxDelayZeroLowLatencyMode) { |
| // Initial keyframe. |
| StartNextDecodeForceKeyframe(); |
| timing_.set_min_playout_delay(TimeDelta::Zero()); |
| timing_.set_max_playout_delay(TimeDelta::Zero()); |
| // Playout delay of 0 implies low-latency rendering. |
| auto frame = test::FakeFrameBuilder() |
| .Id(0) |
| .Time(0) |
| .PlayoutDelay(VideoPlayoutDelay::Minimal()) |
| .AsLast() |
| .Build(); |
| buffer_->InsertFrame(std::move(frame)); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| // Delta frame would normally wait here, but should decode at the pacing rate |
| // in low-latency mode. |
| StartNextDecode(); |
| frame = test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kFps30Rtp) |
| .PlayoutDelay(VideoPlayoutDelay::Minimal()) |
| .AsLast() |
| .Build(); |
| buffer_->InsertFrame(std::move(frame)); |
| // The min/max=0 version of low-latency rendering will result in a large |
| // negative decode wait time, so the frame should be ready right away. |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1))); |
| } |
| |
| INSTANTIATE_TEST_SUITE_P( |
| VideoStreamBufferController, |
| LowLatencyVideoStreamBufferControllerTest, |
| ::testing::Combine( |
| ::testing::Bool(), |
| ::testing::Values( |
| "WebRTC-ZeroPlayoutDelay/min_pacing:16ms,max_decode_queue_size:5/", |
| "WebRTC-ZeroPlayoutDelay/" |
| "min_pacing:16ms,max_decode_queue_size:5/"))); |
| |
| class IncomingTimestampVideoStreamBufferControllerTest |
| : public ::testing::Test, |
| public VideoStreamBufferControllerFixture {}; |
| |
| TEST_P(IncomingTimestampVideoStreamBufferControllerTest, |
| IncomingTimestampOnMarkerBitOnly) { |
| StartNextDecodeForceKeyframe(); |
| EXPECT_CALL(timing_, IncomingTimestampMocked) |
| .Times(field_trials_.IsDisabled("WebRTC-IncomingTimestampOnMarkerBitOnly") |
| ? 3 |
| : 1); |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(0).SpatialLayer(0).Time(0).Build())); |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(1).SpatialLayer(1).Time(0).Build())); |
| buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(2).SpatialLayer(2).Time(0).AsLast().Build())); |
| } |
| |
| INSTANTIATE_TEST_SUITE_P( |
| VideoStreamBufferController, |
| IncomingTimestampVideoStreamBufferControllerTest, |
| ::testing::Combine( |
| ::testing::Bool(), |
| ::testing::Values( |
| "WebRTC-IncomingTimestampOnMarkerBitOnly/Enabled/", |
| "WebRTC-IncomingTimestampOnMarkerBitOnly/Disabled/"))); |
| |
| } // namespace webrtc |
