| /* |
| * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #include "video/frame_buffer_proxy.h" |
| |
| #include <stdint.h> |
| |
| #include <limits> |
| #include <memory> |
| #include <string> |
| #include <utility> |
| #include <vector> |
| |
| #include "absl/types/optional.h" |
| #include "absl/types/variant.h" |
| #include "api/metronome/test/fake_metronome.h" |
| #include "api/units/frequency.h" |
| #include "api/units/time_delta.h" |
| #include "api/units/timestamp.h" |
| #include "api/video/video_content_type.h" |
| #include "api/video/video_timing.h" |
| #include "rtc_base/checks.h" |
| #include "rtc_base/event.h" |
| #include "test/fake_encoded_frame.h" |
| #include "test/gmock.h" |
| #include "test/gtest.h" |
| #include "test/run_loop.h" |
| #include "test/scoped_key_value_config.h" |
| #include "test/time_controller/simulated_time_controller.h" |
| #include "video/decode_synchronizer.h" |
| |
| using ::testing::_; |
| using ::testing::AllOf; |
| using ::testing::Contains; |
| using ::testing::Each; |
| using ::testing::Eq; |
| using ::testing::IsEmpty; |
| using ::testing::Matches; |
| using ::testing::Ne; |
| using ::testing::Not; |
| using ::testing::Optional; |
| using ::testing::Pointee; |
| using ::testing::SizeIs; |
| using ::testing::VariantWith; |
| |
| namespace webrtc { |
| |
| namespace { |
| |
| constexpr size_t kFrameSize = 10; |
| constexpr uint32_t kFps30Rtp = 90000 / 30; |
| constexpr TimeDelta kFps30Delay = 1 / Frequency::Hertz(30); |
| const VideoPlayoutDelay kZeroPlayoutDelay = {0, 0}; |
| constexpr Timestamp kClockStart = Timestamp::Millis(1000); |
| |
| auto TimedOut() { |
| return Optional(VariantWith<TimeDelta>(_)); |
| } |
| |
| auto Frame(testing::Matcher<EncodedFrame> m) { |
| return Optional(VariantWith<std::unique_ptr<EncodedFrame>>(Pointee(m))); |
| } |
| |
| std::unique_ptr<test::FakeEncodedFrame> WithReceiveTimeFromRtpTimestamp( |
| std::unique_ptr<test::FakeEncodedFrame> frame) { |
| if (frame->Timestamp() == 0) { |
| frame->SetReceivedTime(kClockStart.ms()); |
| } else { |
| frame->SetReceivedTime( |
| TimeDelta::Seconds(frame->Timestamp() / 90000.0).ms() + |
| kClockStart.ms()); |
| } |
| return frame; |
| } |
| |
| class VCMReceiveStatisticsCallbackMock : public VCMReceiveStatisticsCallback { |
| public: |
| MOCK_METHOD(void, |
| OnCompleteFrame, |
| (bool is_keyframe, |
| size_t size_bytes, |
| VideoContentType content_type), |
| (override)); |
| MOCK_METHOD(void, OnDroppedFrames, (uint32_t num_dropped), (override)); |
| MOCK_METHOD(void, |
| OnFrameBufferTimingsUpdated, |
| (int max_decode_ms, |
| int current_delay_ms, |
| int target_delay_ms, |
| int jitter_buffer_ms, |
| int min_playout_delay_ms, |
| int render_delay_ms), |
| (override)); |
| MOCK_METHOD(void, |
| OnTimingFrameInfoUpdated, |
| (const TimingFrameInfo& info), |
| (override)); |
| }; |
| |
| bool IsFrameBuffer2Enabled(const FieldTrialsView& field_trials) { |
| return field_trials.Lookup("WebRTC-FrameBuffer3").find("arm:FrameBuffer2") != |
| std::string::npos; |
| } |
| |
| } // namespace |
| |
| constexpr auto kMaxWaitForKeyframe = TimeDelta::Millis(500); |
| constexpr auto kMaxWaitForFrame = TimeDelta::Millis(1500); |
| class FrameBufferProxyFixture |
| : public ::testing::WithParamInterface<std::string>, |
| public FrameSchedulingReceiver { |
| public: |
| FrameBufferProxyFixture() |
| : field_trials_(GetParam()), |
| time_controller_(kClockStart), |
| clock_(time_controller_.GetClock()), |
| decode_queue_(time_controller_.GetTaskQueueFactory()->CreateTaskQueue( |
| "decode_queue", |
| TaskQueueFactory::Priority::NORMAL)), |
| fake_metronome_(time_controller_.GetTaskQueueFactory(), |
| TimeDelta::Millis(16)), |
| decode_sync_(clock_, &fake_metronome_, run_loop_.task_queue()), |
| timing_(clock_, field_trials_), |
| proxy_(FrameBufferProxy::CreateFromFieldTrial(clock_, |
| run_loop_.task_queue(), |
| &timing_, |
| &stats_callback_, |
| &decode_queue_, |
| this, |
| kMaxWaitForKeyframe, |
| kMaxWaitForFrame, |
| &decode_sync_, |
| field_trials_)) { |
| // Avoid starting with negative render times. |
| timing_.set_min_playout_delay(TimeDelta::Millis(10)); |
| |
| ON_CALL(stats_callback_, OnDroppedFrames) |
| .WillByDefault( |
| [this](auto num_dropped) { dropped_frames_ += num_dropped; }); |
| } |
| |
| ~FrameBufferProxyFixture() override { |
| if (proxy_) { |
| proxy_->StopOnWorker(); |
| } |
| fake_metronome_.Stop(); |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| } |
| |
| void OnEncodedFrame(std::unique_ptr<EncodedFrame> frame) override { |
| RTC_DCHECK(frame); |
| SetWaitResult(std::move(frame)); |
| } |
| |
| void OnDecodableFrameTimeout(TimeDelta wait_time) override { |
| SetWaitResult(wait_time); |
| } |
| |
| using WaitResult = |
| absl::variant<std::unique_ptr<EncodedFrame>, TimeDelta /*wait_time*/>; |
| |
| absl::optional<WaitResult> WaitForFrameOrTimeout(TimeDelta wait) { |
| if (wait_result_) { |
| return std::move(wait_result_); |
| } |
| run_loop_.PostTask([&] { time_controller_.AdvanceTime(wait); }); |
| run_loop_.PostTask([&] { |
| if (wait_result_) |
| return; |
| |
| // If run loop posted to a task queue, flush that if there is no result. |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| if (wait_result_) |
| return; |
| |
| run_loop_.PostTask([&] { |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| // Quit if there is no result set. |
| if (!wait_result_) |
| run_loop_.Quit(); |
| }); |
| }); |
| run_loop_.Run(); |
| return std::move(wait_result_); |
| } |
| |
| void StartNextDecode() { |
| ResetLastResult(); |
| proxy_->StartNextDecode(false); |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| } |
| |
| void StartNextDecodeForceKeyframe() { |
| ResetLastResult(); |
| proxy_->StartNextDecode(true); |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| } |
| |
| void ResetLastResult() { wait_result_.reset(); } |
| |
| int dropped_frames() const { return dropped_frames_; } |
| |
| protected: |
| test::ScopedKeyValueConfig field_trials_; |
| GlobalSimulatedTimeController time_controller_; |
| Clock* const clock_; |
| test::RunLoop run_loop_; |
| rtc::TaskQueue decode_queue_; |
| test::FakeMetronome fake_metronome_; |
| DecodeSynchronizer decode_sync_; |
| VCMTiming timing_; |
| |
| ::testing::NiceMock<VCMReceiveStatisticsCallbackMock> stats_callback_; |
| std::unique_ptr<FrameBufferProxy> proxy_; |
| |
| private: |
| void SetWaitResult(WaitResult result) { |
| RTC_DCHECK(!wait_result_); |
| if (absl::holds_alternative<std::unique_ptr<EncodedFrame>>(result)) { |
| RTC_DCHECK(absl::get<std::unique_ptr<EncodedFrame>>(result)); |
| } |
| wait_result_.emplace(std::move(result)); |
| run_loop_.Quit(); |
| } |
| |
| uint32_t dropped_frames_ = 0; |
| absl::optional<WaitResult> wait_result_; |
| }; |
| |
| class FrameBufferProxyTest : public ::testing::Test, |
| public FrameBufferProxyFixture {}; |
| |
| TEST_P(FrameBufferProxyTest, InitialTimeoutAfterKeyframeTimeoutPeriod) { |
| StartNextDecodeForceKeyframe(); |
| // No frame insterted. Timeout expected. |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), TimedOut()); |
| |
| // No new timeout set since receiver has not started new decode. |
| ResetLastResult(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), Eq(absl::nullopt)); |
| |
| // Now that receiver has asked for new frame, a new timeout can occur. |
| StartNextDecodeForceKeyframe(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), TimedOut()); |
| } |
| |
| TEST_P(FrameBufferProxyTest, KeyFramesAreScheduled) { |
| StartNextDecodeForceKeyframe(); |
| time_controller_.AdvanceTime(TimeDelta::Millis(50)); |
| |
| auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build(); |
| proxy_->InsertFrame(std::move(frame)); |
| |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| } |
| |
| TEST_P(FrameBufferProxyTest, DeltaFrameTimeoutAfterKeyframeExtracted) { |
| StartNextDecodeForceKeyframe(); |
| |
| time_controller_.AdvanceTime(TimeDelta::Millis(50)); |
| auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build(); |
| proxy_->InsertFrame(std::move(frame)); |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), |
| Frame(test::WithId(0))); |
| |
| StartNextDecode(); |
| time_controller_.AdvanceTime(TimeDelta::Millis(50)); |
| |
| // Timeouts should now happen at the normal frequency. |
| const int expected_timeouts = 5; |
| for (int i = 0; i < expected_timeouts; ++i) { |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); |
| StartNextDecode(); |
| } |
| } |
| |
| TEST_P(FrameBufferProxyTest, DependantFramesAreScheduled) { |
| StartNextDecodeForceKeyframe(); |
| proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| StartNextDecode(); |
| |
| time_controller_.AdvanceTime(kFps30Delay); |
| proxy_->InsertFrame(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); |
| } |
| |
| TEST_P(FrameBufferProxyTest, SpatialLayersAreScheduled) { |
| StartNextDecodeForceKeyframe(); |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(0).SpatialLayer(0).Time(0).Build())); |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(1).SpatialLayer(1).Time(0).Build())); |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(2).SpatialLayer(2).Time(0).AsLast().Build())); |
| EXPECT_THAT( |
| WaitForFrameOrTimeout(TimeDelta::Zero()), |
| Frame(AllOf(test::WithId(0), test::FrameWithSize(3 * kFrameSize)))); |
| |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(3).Time(kFps30Rtp).SpatialLayer(0).Build())); |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(4).Time(kFps30Rtp).SpatialLayer(1).Build())); |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| .Id(5) |
| .Time(kFps30Rtp) |
| .SpatialLayer(2) |
| .AsLast() |
| .Build())); |
| |
| StartNextDecode(); |
| EXPECT_THAT( |
| WaitForFrameOrTimeout(kFps30Delay * 10), |
| Frame(AllOf(test::WithId(3), test::FrameWithSize(3 * kFrameSize)))); |
| } |
| |
| TEST_P(FrameBufferProxyTest, OutstandingFrameTasksAreCancelledAfterDeletion) { |
| StartNextDecodeForceKeyframe(); |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build())); |
| // Get keyframe. Delta frame should now be scheduled. |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| StartNextDecode(); |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build())); |
| proxy_->StopOnWorker(); |
| // Wait for 2x max wait time. Since we stopped, this should cause no timeouts |
| // or frame-ready callbacks. |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame * 2), Eq(absl::nullopt)); |
| } |
| |
| TEST_P(FrameBufferProxyTest, FramesWaitForDecoderToComplete) { |
| StartNextDecodeForceKeyframe(); |
| |
| // Start with a keyframe. |
| proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| ResetLastResult(); |
| // Insert a delta frame. |
| proxy_->InsertFrame(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build()); |
| |
| // Advancing time should not result in a frame since the scheduler has not |
| // been signalled that we are ready. |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Eq(absl::nullopt)); |
| // Signal ready. |
| StartNextDecode(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); |
| } |
| |
| TEST_P(FrameBufferProxyTest, LateFrameDropped) { |
| StartNextDecodeForceKeyframe(); |
| // F1 |
| // / |
| // F0 --> F2 |
| proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| // Start with a keyframe. |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| StartNextDecode(); |
| |
| // Simulate late F1 which arrives after F2. |
| time_controller_.AdvanceTime(kFps30Delay * 2); |
| proxy_->InsertFrame(test::FakeFrameBuilder() |
| .Id(2) |
| .Time(2 * kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2))); |
| |
| StartNextDecode(); |
| |
| proxy_->InsertFrame(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(1 * kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build()); |
| // Confirm frame 1 is never scheduled by timing out. |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); |
| } |
| |
| TEST_P(FrameBufferProxyTest, FramesFastForwardOnSystemHalt) { |
| StartNextDecodeForceKeyframe(); |
| // F1 |
| // / |
| // F0 --> F2 |
| proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| |
| // Start with a keyframe. |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| time_controller_.AdvanceTime(kFps30Delay); |
| proxy_->InsertFrame(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build()); |
| time_controller_.AdvanceTime(kFps30Delay); |
| proxy_->InsertFrame(test::FakeFrameBuilder() |
| .Id(2) |
| .Time(2 * kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build()); |
| |
| // Halting time should result in F1 being skipped. |
| time_controller_.AdvanceTime(kFps30Delay * 2); |
| StartNextDecode(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2))); |
| EXPECT_EQ(dropped_frames(), 1); |
| } |
| |
| TEST_P(FrameBufferProxyTest, ForceKeyFrame) { |
| StartNextDecodeForceKeyframe(); |
| // Initial keyframe. |
| proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| StartNextDecodeForceKeyframe(); |
| |
| // F2 is the next keyframe, and should be extracted since a keyframe was |
| // forced. |
| proxy_->InsertFrame(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kFps30Rtp) |
| .AsLast() |
| .Refs({0}) |
| .Build()); |
| proxy_->InsertFrame( |
| test::FakeFrameBuilder().Id(2).Time(kFps30Rtp * 2).AsLast().Build()); |
| |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay * 3), Frame(test::WithId(2))); |
| } |
| |
| TEST_P(FrameBufferProxyTest, SlowDecoderDropsTemporalLayers) { |
| StartNextDecodeForceKeyframe(); |
| // 2 temporal layers, at 15fps per layer to make 30fps total. |
| // Decoder is slower than 30fps, so last_frame() will be skipped. |
| // F1 --> F3 --> F5 |
| // / / / |
| // F0 --> F2 --> F4 |
| proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| // Keyframe received. |
| // Don't start next decode until slow delay. |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| time_controller_.AdvanceTime(kFps30Delay); |
| proxy_->InsertFrame(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(1 * kFps30Rtp) |
| .Refs({0}) |
| .AsLast() |
| .Build()); |
| time_controller_.AdvanceTime(kFps30Delay); |
| proxy_->InsertFrame(test::FakeFrameBuilder() |
| .Id(2) |
| .Time(2 * kFps30Rtp) |
| .Refs({0}) |
| .AsLast() |
| .Build()); |
| |
| // Simulate decode taking 3x FPS rate. |
| time_controller_.AdvanceTime(kFps30Delay * 1.5); |
| StartNextDecode(); |
| // F2 is the best frame since decoding was so slow that F1 is too old. |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay * 2), Frame(test::WithId(2))); |
| EXPECT_EQ(dropped_frames(), 1); |
| time_controller_.AdvanceTime(kFps30Delay / 2); |
| |
| proxy_->InsertFrame(test::FakeFrameBuilder() |
| .Id(3) |
| .Time(3 * kFps30Rtp) |
| .Refs({1, 2}) |
| .AsLast() |
| .Build()); |
| time_controller_.AdvanceTime(kFps30Delay / 2); |
| proxy_->InsertFrame(test::FakeFrameBuilder() |
| .Id(4) |
| .Time(4 * kFps30Rtp) |
| .Refs({2}) |
| .AsLast() |
| .Build()); |
| time_controller_.AdvanceTime(kFps30Delay / 2); |
| |
| // F4 is the best frame since decoding was so slow that F1 is too old. |
| time_controller_.AdvanceTime(kFps30Delay); |
| StartNextDecode(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(4))); |
| |
| proxy_->InsertFrame(test::FakeFrameBuilder() |
| .Id(5) |
| .Time(5 * kFps30Rtp) |
| .Refs({3, 4}) |
| .AsLast() |
| .Build()); |
| time_controller_.AdvanceTime(kFps30Delay / 2); |
| |
| // F5 is not decodable since F4 was decoded, so a timeout is expected. |
| time_controller_.AdvanceTime(TimeDelta::Millis(10)); |
| StartNextDecode(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); |
| // TODO(bugs.webrtc.org/13343): This should be 2 dropped frames since frames 1 |
| // and 3 were dropped. However, frame_buffer2 does not mark frame 3 as dropped |
| // which is a bug. Uncomment below when that is fixed for frame_buffer2 is |
| // deleted. |
| // EXPECT_EQ(dropped_frames(), 2); |
| } |
| |
| TEST_P(FrameBufferProxyTest, NewFrameInsertedWhileWaitingToReleaseFrame) { |
| StartNextDecodeForceKeyframe(); |
| // Initial keyframe. |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build())); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| time_controller_.AdvanceTime(kFps30Delay / 2); |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kFps30Rtp) |
| .Refs({0}) |
| .AsLast() |
| .Build())); |
| StartNextDecode(); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Eq(absl::nullopt)); |
| |
| // Scheduler is waiting to deliver Frame 1 now. Insert Frame 2. Frame 1 should |
| // be delivered still. |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| .Id(2) |
| .Time(kFps30Rtp * 2) |
| .Refs({0}) |
| .AsLast() |
| .Build())); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); |
| } |
| |
| TEST_P(FrameBufferProxyTest, SameFrameNotScheduledTwice) { |
| // A frame could be scheduled twice if last_frame() arrive out-of-order but |
| // the older frame is old enough to be fast forwarded. |
| // |
| // 1. F2 arrives and is scheduled. |
| // 2. F3 arrives, but scheduling will not change since F2 is next. |
| // 3. F1 arrives late and scheduling is checked since it is before F2. F1 |
| // fast-forwarded since it is older. |
| // |
| // F2 is the best frame, but should only be scheduled once, followed by F3. |
| StartNextDecodeForceKeyframe(); |
| |
| // First keyframe. |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build())); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Millis(15)), |
| Frame(test::WithId(0))); |
| |
| StartNextDecode(); |
| |
| // Warmup VCMTiming for 30fps. |
| for (int i = 1; i <= 30; ++i) { |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(i).Time(i * kFps30Rtp).AsLast().Build())); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(i))); |
| StartNextDecode(); |
| } |
| |
| // F2 arrives and is scheduled. |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(32).Time(32 * kFps30Rtp).AsLast().Build())); |
| |
| // F3 arrives before F2 is extracted. |
| time_controller_.AdvanceTime(kFps30Delay); |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(33).Time(33 * kFps30Rtp).AsLast().Build())); |
| |
| // F1 arrives and is fast-forwarded since it is too late. |
| // F2 is already scheduled and should not be rescheduled. |
| time_controller_.AdvanceTime(kFps30Delay / 2); |
| proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| test::FakeFrameBuilder().Id(31).Time(31 * kFps30Rtp).AsLast().Build())); |
| |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(32))); |
| StartNextDecode(); |
| |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(33))); |
| StartNextDecode(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); |
| EXPECT_EQ(dropped_frames(), 1); |
| } |
| |
| TEST_P(FrameBufferProxyTest, TestStatsCallback) { |
| EXPECT_CALL(stats_callback_, |
| OnCompleteFrame(true, kFrameSize, VideoContentType::UNSPECIFIED)); |
| EXPECT_CALL(stats_callback_, OnFrameBufferTimingsUpdated); |
| |
| // Fake timing having received decoded frame. |
| timing_.StopDecodeTimer(TimeDelta::Millis(1), clock_->CurrentTime()); |
| StartNextDecodeForceKeyframe(); |
| proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| // Flush stats posted on the decode queue. |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| } |
| |
| TEST_P(FrameBufferProxyTest, FrameCompleteCalledOnceForDuplicateFrame) { |
| EXPECT_CALL(stats_callback_, |
| OnCompleteFrame(true, kFrameSize, VideoContentType::UNSPECIFIED)) |
| .Times(1); |
| |
| StartNextDecodeForceKeyframe(); |
| proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| // Flush stats posted on the decode queue. |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| } |
| |
| TEST_P(FrameBufferProxyTest, FrameCompleteCalledOnceForSingleTemporalUnit) { |
| StartNextDecodeForceKeyframe(); |
| |
| // `OnCompleteFrame` should not be called for the first two frames since they |
| // do not complete the temporal layer. |
| EXPECT_CALL(stats_callback_, OnCompleteFrame(_, _, _)).Times(0); |
| proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).Build()); |
| proxy_->InsertFrame(test::FakeFrameBuilder().Id(1).Time(0).Refs({0}).Build()); |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| // Flush stats posted on the decode queue. |
| ::testing::Mock::VerifyAndClearExpectations(&stats_callback_); |
| |
| // Note that this frame is not marked as a keyframe since the last spatial |
| // layer has dependencies. |
| EXPECT_CALL(stats_callback_, |
| OnCompleteFrame(false, kFrameSize, VideoContentType::UNSPECIFIED)) |
| .Times(1); |
| proxy_->InsertFrame( |
| test::FakeFrameBuilder().Id(2).Time(0).Refs({0, 1}).AsLast().Build()); |
| // Flush stats posted on the decode queue. |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| } |
| |
| TEST_P(FrameBufferProxyTest, FrameCompleteCalledOnceForCompleteTemporalUnit) { |
| // FrameBuffer2 logs the complete frame on the arrival of the last layer. |
| if (IsFrameBuffer2Enabled(field_trials_)) |
| return; |
| StartNextDecodeForceKeyframe(); |
| |
| // `OnCompleteFrame` should not be called for the first two frames since they |
| // do not complete the temporal layer. Frame 1 arrives later, at which time |
| // this frame can finally be considered complete. |
| EXPECT_CALL(stats_callback_, OnCompleteFrame(_, _, _)).Times(0); |
| proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).Build()); |
| proxy_->InsertFrame( |
| test::FakeFrameBuilder().Id(2).Time(0).Refs({0, 1}).AsLast().Build()); |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| // Flush stats posted on the decode queue. |
| ::testing::Mock::VerifyAndClearExpectations(&stats_callback_); |
| |
| EXPECT_CALL(stats_callback_, |
| OnCompleteFrame(false, kFrameSize, VideoContentType::UNSPECIFIED)) |
| .Times(1); |
| proxy_->InsertFrame(test::FakeFrameBuilder().Id(1).Time(0).Refs({0}).Build()); |
| // Flush stats posted on the decode queue. |
| time_controller_.AdvanceTime(TimeDelta::Zero()); |
| } |
| |
| // Note: This test takes a long time to run if the fake metronome is active. |
| // Since the test needs to wait for the timestamp to rollover, it has a fake |
| // delay of around 6.5 hours. Even though time is simulated, this will be |
| // around 1,500,000 metronome tick invocations. |
| TEST_P(FrameBufferProxyTest, NextFrameWithOldTimestamp) { |
| // Test inserting 31 frames and pause the stream for a long time before |
| // frame 32. |
| StartNextDecodeForceKeyframe(); |
| constexpr uint32_t kBaseRtp = std::numeric_limits<uint32_t>::max() / 2; |
| |
| // First keyframe. The receive time must be explicitly set in this test since |
| // the RTP derived time used in all tests does not work when the long pause |
| // happens later in the test. |
| proxy_->InsertFrame(test::FakeFrameBuilder() |
| .Id(0) |
| .Time(kBaseRtp) |
| .ReceivedTime(clock_->CurrentTime()) |
| .AsLast() |
| .Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(0))); |
| |
| // 1 more frame to warmup VCMTiming for 30fps. |
| StartNextDecode(); |
| proxy_->InsertFrame(test::FakeFrameBuilder() |
| .Id(1) |
| .Time(kBaseRtp + kFps30Rtp) |
| .ReceivedTime(clock_->CurrentTime()) |
| .AsLast() |
| .Build()); |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); |
| |
| // Pause the stream for such a long time it incurs an RTP timestamp rollover |
| // by over half. |
| constexpr uint32_t kLastRtp = kBaseRtp + kFps30Rtp; |
| constexpr uint32_t kRolloverRtp = |
| kLastRtp + std::numeric_limits<uint32_t>::max() / 2 + 1; |
| constexpr Frequency kRtpHz = Frequency::KiloHertz(90); |
| // Pause for corresponding delay such that RTP timestamp would increase this |
| // much at 30fps. |
| constexpr TimeDelta kRolloverDelay = |
| (std::numeric_limits<uint32_t>::max() / 2 + 1) / kRtpHz; |
| |
| // Avoid timeout being set while waiting for the frame and before the receiver |
| // is ready. |
| ResetLastResult(); |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), Eq(absl::nullopt)); |
| time_controller_.AdvanceTime(kRolloverDelay - kMaxWaitForFrame); |
| StartNextDecode(); |
| proxy_->InsertFrame(test::FakeFrameBuilder() |
| .Id(2) |
| .Time(kRolloverRtp) |
| .ReceivedTime(clock_->CurrentTime()) |
| .AsLast() |
| .Build()); |
| // FrameBuffer2 drops the frame, while FrameBuffer3 will continue the stream. |
| if (!IsFrameBuffer2Enabled(field_trials_)) { |
| EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2))); |
| } else { |
| EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); |
| } |
| } |
| |
| INSTANTIATE_TEST_SUITE_P( |
| FrameBufferProxy, |
| FrameBufferProxyTest, |
| ::testing::Values("WebRTC-FrameBuffer3/arm:FrameBuffer2/", |
| "WebRTC-FrameBuffer3/arm:FrameBuffer3/", |
| "WebRTC-FrameBuffer3/arm:SyncDecoding/")); |
| |
| class LowLatencyFrameBufferProxyTest : public ::testing::Test, |
| public FrameBufferProxyFixture {}; |
| |
| TEST_P(LowLatencyFrameBufferProxyTest, |
| FramesDecodedInstantlyWithLowLatencyRendering) { |
| // Initial keyframe. |
| StartNextDecodeForceKeyframe(); |
| timing_.set_min_playout_delay(TimeDelta::Zero()); |
| timing_.set_max_playout_delay(TimeDelta::Millis(10)); |
| auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build(); |
| // Playout delay of 0 implies low-latency rendering. |
| frame->SetPlayoutDelay({0, 10}); |
| proxy_->InsertFrame(std::move(frame)); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| // Delta frame would normally wait here, but should decode at the pacing rate |
| // in low-latency mode. |
| StartNextDecode(); |
| frame = test::FakeFrameBuilder().Id(1).Time(kFps30Rtp).AsLast().Build(); |
| frame->SetPlayoutDelay({0, 10}); |
| proxy_->InsertFrame(std::move(frame)); |
| // Pacing is set to 16ms in the field trial so we should not decode yet. |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Eq(absl::nullopt)); |
| time_controller_.AdvanceTime(TimeDelta::Millis(16)); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1))); |
| } |
| |
| TEST_P(LowLatencyFrameBufferProxyTest, ZeroPlayoutDelayFullQueue) { |
| // Initial keyframe. |
| StartNextDecodeForceKeyframe(); |
| timing_.set_min_playout_delay(TimeDelta::Zero()); |
| timing_.set_max_playout_delay(TimeDelta::Millis(10)); |
| auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build(); |
| // Playout delay of 0 implies low-latency rendering. |
| frame->SetPlayoutDelay({0, 10}); |
| proxy_->InsertFrame(std::move(frame)); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| // Queue up 5 frames (configured max queue size for 0-playout delay pacing). |
| for (int id = 1; id <= 6; ++id) { |
| frame = |
| test::FakeFrameBuilder().Id(id).Time(kFps30Rtp * id).AsLast().Build(); |
| frame->SetPlayoutDelay({0, 10}); |
| proxy_->InsertFrame(std::move(frame)); |
| } |
| |
| // The queue is at its max size for zero playout delay pacing, so the pacing |
| // should be ignored and the next frame should be decoded instantly. |
| StartNextDecode(); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1))); |
| } |
| |
| TEST_P(LowLatencyFrameBufferProxyTest, MinMaxDelayZeroLowLatencyMode) { |
| // Initial keyframe. |
| StartNextDecodeForceKeyframe(); |
| timing_.set_min_playout_delay(TimeDelta::Zero()); |
| timing_.set_max_playout_delay(TimeDelta::Zero()); |
| auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build(); |
| // Playout delay of 0 implies low-latency rendering. |
| frame->SetPlayoutDelay({0, 0}); |
| proxy_->InsertFrame(std::move(frame)); |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| |
| // Delta frame would normally wait here, but should decode at the pacing rate |
| // in low-latency mode. |
| StartNextDecode(); |
| frame = test::FakeFrameBuilder().Id(1).Time(kFps30Rtp).AsLast().Build(); |
| frame->SetPlayoutDelay({0, 0}); |
| proxy_->InsertFrame(std::move(frame)); |
| // The min/max=0 version of low-latency rendering will result in a large |
| // negative decode wait time, so the frame should be ready right away. |
| EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1))); |
| } |
| |
| INSTANTIATE_TEST_SUITE_P( |
| FrameBufferProxy, |
| LowLatencyFrameBufferProxyTest, |
| ::testing::Values( |
| "WebRTC-FrameBuffer3/arm:FrameBuffer2/" |
| "WebRTC-ZeroPlayoutDelay/min_pacing:16ms,max_decode_queue_size:5/", |
| "WebRTC-FrameBuffer3/arm:FrameBuffer3/" |
| "WebRTC-ZeroPlayoutDelay/min_pacing:16ms,max_decode_queue_size:5/", |
| "WebRTC-FrameBuffer3/arm:SyncDecoding/" |
| "WebRTC-ZeroPlayoutDelay/min_pacing:16ms,max_decode_queue_size:5/")); |
| |
| } // namespace webrtc |
