Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 11 | #include "video/video_stream_buffer_controller.h" |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 12 | |
| 13 | #include <stdint.h> |
| 14 | |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 15 | #include <limits> |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 16 | #include <memory> |
Florent Castelli | 8037fc6 | 2024-08-29 13:00:40 | [diff] [blame] | 17 | #include <optional> |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 18 | #include <string> |
Evan Shrubsole | a006ba1 | 2022-09-05 10:09:08 | [diff] [blame] | 19 | #include <tuple> |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 20 | #include <utility> |
| 21 | #include <vector> |
| 22 | |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 23 | #include "absl/types/variant.h" |
Evan Shrubsole | 6cd6d8e | 2022-02-11 14:30:26 | [diff] [blame] | 24 | #include "api/metronome/test/fake_metronome.h" |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 25 | #include "api/units/frequency.h" |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 26 | #include "api/units/time_delta.h" |
| 27 | #include "api/units/timestamp.h" |
| 28 | #include "api/video/video_content_type.h" |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 29 | #include "api/video/video_timing.h" |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 30 | #include "rtc_base/checks.h" |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 31 | #include "test/fake_encoded_frame.h" |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 32 | #include "test/gmock.h" |
| 33 | #include "test/gtest.h" |
Jonas Oreland | 8ca0613 | 2022-03-14 11:52:48 | [diff] [blame] | 34 | #include "test/scoped_key_value_config.h" |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 35 | #include "test/time_controller/simulated_time_controller.h" |
Evan Shrubsole | 6cd6d8e | 2022-02-11 14:30:26 | [diff] [blame] | 36 | #include "video/decode_synchronizer.h" |
Evan Shrubsole | a006ba1 | 2022-09-05 10:09:08 | [diff] [blame] | 37 | #include "video/task_queue_frame_decode_scheduler.h" |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 38 | |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 39 | using ::testing::_; |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 40 | using ::testing::AllOf; |
| 41 | using ::testing::Contains; |
| 42 | using ::testing::Each; |
| 43 | using ::testing::Eq; |
| 44 | using ::testing::IsEmpty; |
| 45 | using ::testing::Matches; |
| 46 | using ::testing::Ne; |
| 47 | using ::testing::Not; |
| 48 | using ::testing::Optional; |
| 49 | using ::testing::Pointee; |
| 50 | using ::testing::SizeIs; |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 51 | using ::testing::VariantWith; |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 52 | |
| 53 | namespace webrtc { |
| 54 | |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 55 | namespace { |
| 56 | |
| 57 | constexpr size_t kFrameSize = 10; |
| 58 | constexpr uint32_t kFps30Rtp = 90000 / 30; |
| 59 | constexpr TimeDelta kFps30Delay = 1 / Frequency::Hertz(30); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 60 | constexpr Timestamp kClockStart = Timestamp::Millis(1000); |
| 61 | |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 62 | auto TimedOut() { |
| 63 | return Optional(VariantWith<TimeDelta>(_)); |
| 64 | } |
| 65 | |
| 66 | auto Frame(testing::Matcher<EncodedFrame> m) { |
| 67 | return Optional(VariantWith<std::unique_ptr<EncodedFrame>>(Pointee(m))); |
| 68 | } |
| 69 | |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 70 | std::unique_ptr<test::FakeEncodedFrame> WithReceiveTimeFromRtpTimestamp( |
| 71 | std::unique_ptr<test::FakeEncodedFrame> frame) { |
Danil Chapovalov | 9c58483 | 2023-09-18 13:48:49 | [diff] [blame] | 72 | if (frame->RtpTimestamp() == 0) { |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 73 | frame->SetReceivedTime(kClockStart.ms()); |
| 74 | } else { |
| 75 | frame->SetReceivedTime( |
Danil Chapovalov | 9c58483 | 2023-09-18 13:48:49 | [diff] [blame] | 76 | TimeDelta::Seconds(frame->RtpTimestamp() / 90000.0).ms() + |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 77 | kClockStart.ms()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 78 | } |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 79 | return frame; |
| 80 | } |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 81 | |
philipel | 7446b60 | 2022-10-06 13:49:17 | [diff] [blame] | 82 | class VCMTimingTest : public VCMTiming { |
| 83 | public: |
| 84 | using VCMTiming::VCMTiming; |
| 85 | void IncomingTimestamp(uint32_t rtp_timestamp, |
| 86 | Timestamp last_packet_time) override { |
| 87 | IncomingTimestampMocked(rtp_timestamp, last_packet_time); |
| 88 | VCMTiming::IncomingTimestamp(rtp_timestamp, last_packet_time); |
| 89 | } |
| 90 | |
| 91 | MOCK_METHOD(void, |
| 92 | IncomingTimestampMocked, |
| 93 | (uint32_t rtp_timestamp, Timestamp last_packet_time), |
| 94 | ()); |
| 95 | }; |
| 96 | |
Rasmus Brandt | 39250a4 | 2023-05-09 12:33:43 | [diff] [blame] | 97 | class VideoStreamBufferControllerStatsObserverMock |
| 98 | : public VideoStreamBufferControllerStatsObserver { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 99 | public: |
| 100 | MOCK_METHOD(void, |
| 101 | OnCompleteFrame, |
| 102 | (bool is_keyframe, |
| 103 | size_t size_bytes, |
| 104 | VideoContentType content_type), |
| 105 | (override)); |
| 106 | MOCK_METHOD(void, OnDroppedFrames, (uint32_t num_dropped), (override)); |
| 107 | MOCK_METHOD(void, |
Rasmus Brandt | 621cb29 | 2023-05-24 11:29:14 | [diff] [blame] | 108 | OnDecodableFrame, |
Rasmus Brandt | f0820ff | 2023-05-25 07:37:16 | [diff] [blame] | 109 | (TimeDelta jitter_buffer_delay, |
| 110 | TimeDelta target_delay, |
| 111 | TimeDelta minimum_delay), |
Rasmus Brandt | 621cb29 | 2023-05-24 11:29:14 | [diff] [blame] | 112 | (override)); |
| 113 | MOCK_METHOD(void, |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 114 | OnFrameBufferTimingsUpdated, |
Rasmus Brandt | 621cb29 | 2023-05-24 11:29:14 | [diff] [blame] | 115 | (int estimated_max_decode_time_ms, |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 116 | int current_delay_ms, |
| 117 | int target_delay_ms, |
Rasmus Brandt | 621cb29 | 2023-05-24 11:29:14 | [diff] [blame] | 118 | int jitter_delay_ms, |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 119 | int min_playout_delay_ms, |
| 120 | int render_delay_ms), |
| 121 | (override)); |
| 122 | MOCK_METHOD(void, |
| 123 | OnTimingFrameInfoUpdated, |
| 124 | (const TimingFrameInfo& info), |
| 125 | (override)); |
| 126 | }; |
Evan Shrubsole | d4ff12f | 2022-03-01 09:16:11 | [diff] [blame] | 127 | |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 128 | } // namespace |
| 129 | |
| 130 | constexpr auto kMaxWaitForKeyframe = TimeDelta::Millis(500); |
| 131 | constexpr auto kMaxWaitForFrame = TimeDelta::Millis(1500); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 132 | class VideoStreamBufferControllerFixture |
Evan Shrubsole | a006ba1 | 2022-09-05 10:09:08 | [diff] [blame] | 133 | : public ::testing::WithParamInterface<std::tuple<bool, std::string>>, |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 134 | public FrameSchedulingReceiver { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 135 | public: |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 136 | VideoStreamBufferControllerFixture() |
Evan Shrubsole | a006ba1 | 2022-09-05 10:09:08 | [diff] [blame] | 137 | : sync_decoding_(std::get<0>(GetParam())), |
| 138 | field_trials_(std::get<1>(GetParam())), |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 139 | time_controller_(kClockStart), |
| 140 | clock_(time_controller_.GetClock()), |
Markus Handell | be400e4 | 2022-11-08 11:14:23 | [diff] [blame] | 141 | fake_metronome_(TimeDelta::Millis(16)), |
Evan Shrubsole | 214cab5 | 2022-08-16 09:48:23 | [diff] [blame] | 142 | decode_sync_(clock_, |
| 143 | &fake_metronome_, |
| 144 | time_controller_.GetMainThread()), |
Jonas Oreland | e02f9ee | 2022-03-25 11:43:14 | [diff] [blame] | 145 | timing_(clock_, field_trials_), |
Evan Shrubsole | a006ba1 | 2022-09-05 10:09:08 | [diff] [blame] | 146 | buffer_(std::make_unique<VideoStreamBufferController>( |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 147 | clock_, |
Evan Shrubsole | 214cab5 | 2022-08-16 09:48:23 | [diff] [blame] | 148 | time_controller_.GetMainThread(), |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 149 | &timing_, |
| 150 | &stats_callback_, |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 151 | this, |
| 152 | kMaxWaitForKeyframe, |
| 153 | kMaxWaitForFrame, |
Evan Shrubsole | a006ba1 | 2022-09-05 10:09:08 | [diff] [blame] | 154 | sync_decoding_ ? decode_sync_.CreateSynchronizedFrameScheduler() |
| 155 | : std::make_unique<TaskQueueFrameDecodeScheduler>( |
| 156 | clock_, |
| 157 | time_controller_.GetMainThread()), |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 158 | field_trials_)) { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 159 | // Avoid starting with negative render times. |
Evan Shrubsole | d6cdf80 | 2022-03-02 14:13:55 | [diff] [blame] | 160 | timing_.set_min_playout_delay(TimeDelta::Millis(10)); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 161 | |
| 162 | ON_CALL(stats_callback_, OnDroppedFrames) |
| 163 | .WillByDefault( |
| 164 | [this](auto num_dropped) { dropped_frames_ += num_dropped; }); |
| 165 | } |
| 166 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 167 | ~VideoStreamBufferControllerFixture() override { |
| 168 | if (buffer_) { |
Evan Shrubsole | 214cab5 | 2022-08-16 09:48:23 | [diff] [blame] | 169 | buffer_->Stop(); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 170 | } |
Evan Shrubsole | 6cd6d8e | 2022-02-11 14:30:26 | [diff] [blame] | 171 | time_controller_.AdvanceTime(TimeDelta::Zero()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 172 | } |
| 173 | |
| 174 | void OnEncodedFrame(std::unique_ptr<EncodedFrame> frame) override { |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 175 | RTC_DCHECK(frame); |
| 176 | SetWaitResult(std::move(frame)); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 177 | } |
| 178 | |
| 179 | void OnDecodableFrameTimeout(TimeDelta wait_time) override { |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 180 | SetWaitResult(wait_time); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 181 | } |
| 182 | |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 183 | using WaitResult = |
| 184 | absl::variant<std::unique_ptr<EncodedFrame>, TimeDelta /*wait_time*/>; |
| 185 | |
Florent Castelli | 8037fc6 | 2024-08-29 13:00:40 | [diff] [blame] | 186 | std::optional<WaitResult> WaitForFrameOrTimeout(TimeDelta wait) { |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 187 | if (wait_result_) { |
| 188 | return std::move(wait_result_); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 189 | } |
Evan Shrubsole | 214cab5 | 2022-08-16 09:48:23 | [diff] [blame] | 190 | time_controller_.AdvanceTime(TimeDelta::Zero()); |
| 191 | if (wait_result_) { |
| 192 | return std::move(wait_result_); |
| 193 | } |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 194 | |
Evan Shrubsole | 214cab5 | 2022-08-16 09:48:23 | [diff] [blame] | 195 | Timestamp now = clock_->CurrentTime(); |
| 196 | // TODO(bugs.webrtc.org/13756): Remove this when rtc::Thread uses uses |
| 197 | // Timestamp instead of an integer milliseconds. This extra wait is needed |
| 198 | // for some tests that use the metronome. This is due to rounding |
| 199 | // milliseconds, affecting the precision of simulated time controller uses |
| 200 | // when posting tasks from threads. |
| 201 | TimeDelta potential_extra_wait = |
| 202 | Timestamp::Millis((now + wait).ms()) - (now + wait); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 203 | |
Evan Shrubsole | 214cab5 | 2022-08-16 09:48:23 | [diff] [blame] | 204 | time_controller_.AdvanceTime(wait); |
| 205 | if (potential_extra_wait > TimeDelta::Zero()) { |
| 206 | time_controller_.AdvanceTime(potential_extra_wait); |
| 207 | } |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 208 | return std::move(wait_result_); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 209 | } |
| 210 | |
| 211 | void StartNextDecode() { |
| 212 | ResetLastResult(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 213 | buffer_->StartNextDecode(false); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 214 | } |
| 215 | |
| 216 | void StartNextDecodeForceKeyframe() { |
| 217 | ResetLastResult(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 218 | buffer_->StartNextDecode(true); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 219 | } |
| 220 | |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 221 | void ResetLastResult() { wait_result_.reset(); } |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 222 | |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 223 | int dropped_frames() const { return dropped_frames_; } |
| 224 | |
| 225 | protected: |
Evan Shrubsole | a006ba1 | 2022-09-05 10:09:08 | [diff] [blame] | 226 | const bool sync_decoding_; |
Jonas Oreland | 8ca0613 | 2022-03-14 11:52:48 | [diff] [blame] | 227 | test::ScopedKeyValueConfig field_trials_; |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 228 | GlobalSimulatedTimeController time_controller_; |
| 229 | Clock* const clock_; |
Evan Shrubsole | 6cd6d8e | 2022-02-11 14:30:26 | [diff] [blame] | 230 | test::FakeMetronome fake_metronome_; |
| 231 | DecodeSynchronizer decode_sync_; |
Evan Shrubsole | 6cd6d8e | 2022-02-11 14:30:26 | [diff] [blame] | 232 | |
philipel | 7446b60 | 2022-10-06 13:49:17 | [diff] [blame] | 233 | ::testing::NiceMock<VCMTimingTest> timing_; |
Rasmus Brandt | 39250a4 | 2023-05-09 12:33:43 | [diff] [blame] | 234 | ::testing::NiceMock<VideoStreamBufferControllerStatsObserverMock> |
| 235 | stats_callback_; |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 236 | std::unique_ptr<VideoStreamBufferController> buffer_; |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 237 | |
| 238 | private: |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 239 | void SetWaitResult(WaitResult result) { |
| 240 | RTC_DCHECK(!wait_result_); |
| 241 | if (absl::holds_alternative<std::unique_ptr<EncodedFrame>>(result)) { |
| 242 | RTC_DCHECK(absl::get<std::unique_ptr<EncodedFrame>>(result)); |
| 243 | } |
| 244 | wait_result_.emplace(std::move(result)); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 245 | } |
| 246 | |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 247 | uint32_t dropped_frames_ = 0; |
Florent Castelli | 8037fc6 | 2024-08-29 13:00:40 | [diff] [blame] | 248 | std::optional<WaitResult> wait_result_; |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 249 | }; |
| 250 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 251 | class VideoStreamBufferControllerTest |
| 252 | : public ::testing::Test, |
| 253 | public VideoStreamBufferControllerFixture {}; |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 254 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 255 | TEST_P(VideoStreamBufferControllerTest, |
| 256 | InitialTimeoutAfterKeyframeTimeoutPeriod) { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 257 | StartNextDecodeForceKeyframe(); |
Evan Shrubsole | 4c26932 | 2022-08-02 08:57:04 | [diff] [blame] | 258 | // No frame inserted. Timeout expected. |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 259 | EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), TimedOut()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 260 | |
| 261 | // No new timeout set since receiver has not started new decode. |
| 262 | ResetLastResult(); |
Florent Castelli | 8037fc6 | 2024-08-29 13:00:40 | [diff] [blame] | 263 | EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), Eq(std::nullopt)); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 264 | |
| 265 | // Now that receiver has asked for new frame, a new timeout can occur. |
| 266 | StartNextDecodeForceKeyframe(); |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 267 | EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), TimedOut()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 268 | } |
| 269 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 270 | TEST_P(VideoStreamBufferControllerTest, KeyFramesAreScheduled) { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 271 | StartNextDecodeForceKeyframe(); |
| 272 | time_controller_.AdvanceTime(TimeDelta::Millis(50)); |
| 273 | |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 274 | auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 275 | buffer_->InsertFrame(std::move(frame)); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 276 | |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 277 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 278 | } |
| 279 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 280 | TEST_P(VideoStreamBufferControllerTest, |
| 281 | DeltaFrameTimeoutAfterKeyframeExtracted) { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 282 | StartNextDecodeForceKeyframe(); |
| 283 | |
| 284 | time_controller_.AdvanceTime(TimeDelta::Millis(50)); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 285 | auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 286 | buffer_->InsertFrame(std::move(frame)); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 287 | EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), |
| 288 | Frame(test::WithId(0))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 289 | |
| 290 | StartNextDecode(); |
| 291 | time_controller_.AdvanceTime(TimeDelta::Millis(50)); |
| 292 | |
| 293 | // Timeouts should now happen at the normal frequency. |
| 294 | const int expected_timeouts = 5; |
| 295 | for (int i = 0; i < expected_timeouts; ++i) { |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 296 | EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 297 | StartNextDecode(); |
| 298 | } |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 299 | } |
| 300 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 301 | TEST_P(VideoStreamBufferControllerTest, DependantFramesAreScheduled) { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 302 | StartNextDecodeForceKeyframe(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 303 | buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 304 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 305 | |
| 306 | StartNextDecode(); |
| 307 | |
| 308 | time_controller_.AdvanceTime(kFps30Delay); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 309 | buffer_->InsertFrame(test::FakeFrameBuilder() |
| 310 | .Id(1) |
| 311 | .Time(kFps30Rtp) |
| 312 | .AsLast() |
| 313 | .Refs({0}) |
| 314 | .Build()); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 315 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 316 | } |
| 317 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 318 | TEST_P(VideoStreamBufferControllerTest, SpatialLayersAreScheduled) { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 319 | StartNextDecodeForceKeyframe(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 320 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 321 | test::FakeFrameBuilder().Id(0).SpatialLayer(0).Time(0).Build())); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 322 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 323 | test::FakeFrameBuilder().Id(1).SpatialLayer(1).Time(0).Build())); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 324 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 325 | test::FakeFrameBuilder().Id(2).SpatialLayer(2).Time(0).AsLast().Build())); |
| 326 | EXPECT_THAT( |
| 327 | WaitForFrameOrTimeout(TimeDelta::Zero()), |
| 328 | Frame(AllOf(test::WithId(0), test::FrameWithSize(3 * kFrameSize)))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 329 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 330 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 331 | test::FakeFrameBuilder().Id(3).Time(kFps30Rtp).SpatialLayer(0).Build())); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 332 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 333 | test::FakeFrameBuilder().Id(4).Time(kFps30Rtp).SpatialLayer(1).Build())); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 334 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| 335 | .Id(5) |
| 336 | .Time(kFps30Rtp) |
| 337 | .SpatialLayer(2) |
| 338 | .AsLast() |
| 339 | .Build())); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 340 | |
| 341 | StartNextDecode(); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 342 | EXPECT_THAT( |
| 343 | WaitForFrameOrTimeout(kFps30Delay * 10), |
| 344 | Frame(AllOf(test::WithId(3), test::FrameWithSize(3 * kFrameSize)))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 345 | } |
| 346 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 347 | TEST_P(VideoStreamBufferControllerTest, |
| 348 | OutstandingFrameTasksAreCancelledAfterDeletion) { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 349 | StartNextDecodeForceKeyframe(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 350 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 351 | test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build())); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 352 | // Get keyframe. Delta frame should now be scheduled. |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 353 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 354 | |
| 355 | StartNextDecode(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 356 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| 357 | .Id(1) |
| 358 | .Time(kFps30Rtp) |
| 359 | .AsLast() |
| 360 | .Refs({0}) |
| 361 | .Build())); |
Evan Shrubsole | 214cab5 | 2022-08-16 09:48:23 | [diff] [blame] | 362 | buffer_->Stop(); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 363 | // Wait for 2x max wait time. Since we stopped, this should cause no timeouts |
| 364 | // or frame-ready callbacks. |
Florent Castelli | 8037fc6 | 2024-08-29 13:00:40 | [diff] [blame] | 365 | EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame * 2), Eq(std::nullopt)); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 366 | } |
| 367 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 368 | TEST_P(VideoStreamBufferControllerTest, FramesWaitForDecoderToComplete) { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 369 | StartNextDecodeForceKeyframe(); |
| 370 | |
| 371 | // Start with a keyframe. |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 372 | buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 373 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 374 | |
| 375 | ResetLastResult(); |
| 376 | // Insert a delta frame. |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 377 | buffer_->InsertFrame(test::FakeFrameBuilder() |
| 378 | .Id(1) |
| 379 | .Time(kFps30Rtp) |
| 380 | .AsLast() |
| 381 | .Refs({0}) |
| 382 | .Build()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 383 | |
| 384 | // Advancing time should not result in a frame since the scheduler has not |
| 385 | // been signalled that we are ready. |
Florent Castelli | 8037fc6 | 2024-08-29 13:00:40 | [diff] [blame] | 386 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Eq(std::nullopt)); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 387 | // Signal ready. |
| 388 | StartNextDecode(); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 389 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 390 | } |
| 391 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 392 | TEST_P(VideoStreamBufferControllerTest, LateFrameDropped) { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 393 | StartNextDecodeForceKeyframe(); |
| 394 | // F1 |
| 395 | // / |
| 396 | // F0 --> F2 |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 397 | buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 398 | // Start with a keyframe. |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 399 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 400 | |
| 401 | StartNextDecode(); |
| 402 | |
| 403 | // Simulate late F1 which arrives after F2. |
| 404 | time_controller_.AdvanceTime(kFps30Delay * 2); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 405 | buffer_->InsertFrame(test::FakeFrameBuilder() |
| 406 | .Id(2) |
| 407 | .Time(2 * kFps30Rtp) |
| 408 | .AsLast() |
| 409 | .Refs({0}) |
| 410 | .Build()); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 411 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 412 | |
| 413 | StartNextDecode(); |
| 414 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 415 | buffer_->InsertFrame(test::FakeFrameBuilder() |
| 416 | .Id(1) |
| 417 | .Time(1 * kFps30Rtp) |
| 418 | .AsLast() |
| 419 | .Refs({0}) |
| 420 | .Build()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 421 | // Confirm frame 1 is never scheduled by timing out. |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 422 | EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 423 | } |
| 424 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 425 | TEST_P(VideoStreamBufferControllerTest, FramesFastForwardOnSystemHalt) { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 426 | StartNextDecodeForceKeyframe(); |
| 427 | // F1 |
| 428 | // / |
| 429 | // F0 --> F2 |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 430 | buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 431 | |
| 432 | // Start with a keyframe. |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 433 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 434 | |
| 435 | time_controller_.AdvanceTime(kFps30Delay); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 436 | buffer_->InsertFrame(test::FakeFrameBuilder() |
| 437 | .Id(1) |
| 438 | .Time(kFps30Rtp) |
| 439 | .AsLast() |
| 440 | .Refs({0}) |
| 441 | .Build()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 442 | time_controller_.AdvanceTime(kFps30Delay); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 443 | buffer_->InsertFrame(test::FakeFrameBuilder() |
| 444 | .Id(2) |
| 445 | .Time(2 * kFps30Rtp) |
| 446 | .AsLast() |
| 447 | .Refs({0}) |
| 448 | .Build()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 449 | |
| 450 | // Halting time should result in F1 being skipped. |
| 451 | time_controller_.AdvanceTime(kFps30Delay * 2); |
| 452 | StartNextDecode(); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 453 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 454 | EXPECT_EQ(dropped_frames(), 1); |
| 455 | } |
| 456 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 457 | TEST_P(VideoStreamBufferControllerTest, ForceKeyFrame) { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 458 | StartNextDecodeForceKeyframe(); |
| 459 | // Initial keyframe. |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 460 | buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 461 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 462 | |
| 463 | StartNextDecodeForceKeyframe(); |
| 464 | |
| 465 | // F2 is the next keyframe, and should be extracted since a keyframe was |
| 466 | // forced. |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 467 | buffer_->InsertFrame(test::FakeFrameBuilder() |
| 468 | .Id(1) |
| 469 | .Time(kFps30Rtp) |
| 470 | .AsLast() |
| 471 | .Refs({0}) |
| 472 | .Build()); |
| 473 | buffer_->InsertFrame( |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 474 | test::FakeFrameBuilder().Id(2).Time(kFps30Rtp * 2).AsLast().Build()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 475 | |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 476 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay * 3), Frame(test::WithId(2))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 477 | } |
| 478 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 479 | TEST_P(VideoStreamBufferControllerTest, SlowDecoderDropsTemporalLayers) { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 480 | StartNextDecodeForceKeyframe(); |
| 481 | // 2 temporal layers, at 15fps per layer to make 30fps total. |
| 482 | // Decoder is slower than 30fps, so last_frame() will be skipped. |
| 483 | // F1 --> F3 --> F5 |
| 484 | // / / / |
| 485 | // F0 --> F2 --> F4 |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 486 | buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 487 | // Keyframe received. |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 488 | // Don't start next decode until slow delay. |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 489 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 490 | |
| 491 | time_controller_.AdvanceTime(kFps30Delay); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 492 | buffer_->InsertFrame(test::FakeFrameBuilder() |
| 493 | .Id(1) |
| 494 | .Time(1 * kFps30Rtp) |
| 495 | .Refs({0}) |
| 496 | .AsLast() |
| 497 | .Build()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 498 | time_controller_.AdvanceTime(kFps30Delay); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 499 | buffer_->InsertFrame(test::FakeFrameBuilder() |
| 500 | .Id(2) |
| 501 | .Time(2 * kFps30Rtp) |
| 502 | .Refs({0}) |
| 503 | .AsLast() |
| 504 | .Build()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 505 | |
| 506 | // Simulate decode taking 3x FPS rate. |
| 507 | time_controller_.AdvanceTime(kFps30Delay * 1.5); |
| 508 | StartNextDecode(); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 509 | // F2 is the best frame since decoding was so slow that F1 is too old. |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 510 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay * 2), Frame(test::WithId(2))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 511 | EXPECT_EQ(dropped_frames(), 1); |
| 512 | time_controller_.AdvanceTime(kFps30Delay / 2); |
| 513 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 514 | buffer_->InsertFrame(test::FakeFrameBuilder() |
| 515 | .Id(3) |
| 516 | .Time(3 * kFps30Rtp) |
| 517 | .Refs({1, 2}) |
| 518 | .AsLast() |
| 519 | .Build()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 520 | time_controller_.AdvanceTime(kFps30Delay / 2); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 521 | buffer_->InsertFrame(test::FakeFrameBuilder() |
| 522 | .Id(4) |
| 523 | .Time(4 * kFps30Rtp) |
| 524 | .Refs({2}) |
| 525 | .AsLast() |
| 526 | .Build()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 527 | time_controller_.AdvanceTime(kFps30Delay / 2); |
| 528 | |
| 529 | // F4 is the best frame since decoding was so slow that F1 is too old. |
| 530 | time_controller_.AdvanceTime(kFps30Delay); |
| 531 | StartNextDecode(); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 532 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(4))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 533 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 534 | buffer_->InsertFrame(test::FakeFrameBuilder() |
| 535 | .Id(5) |
| 536 | .Time(5 * kFps30Rtp) |
| 537 | .Refs({3, 4}) |
| 538 | .AsLast() |
| 539 | .Build()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 540 | time_controller_.AdvanceTime(kFps30Delay / 2); |
| 541 | |
| 542 | // F5 is not decodable since F4 was decoded, so a timeout is expected. |
| 543 | time_controller_.AdvanceTime(TimeDelta::Millis(10)); |
| 544 | StartNextDecode(); |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 545 | EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 546 | // TODO(bugs.webrtc.org/13343): This should be 2 dropped frames since frames 1 |
| 547 | // and 3 were dropped. However, frame_buffer2 does not mark frame 3 as dropped |
| 548 | // which is a bug. Uncomment below when that is fixed for frame_buffer2 is |
| 549 | // deleted. |
| 550 | // EXPECT_EQ(dropped_frames(), 2); |
| 551 | } |
| 552 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 553 | TEST_P(VideoStreamBufferControllerTest, |
| 554 | NewFrameInsertedWhileWaitingToReleaseFrame) { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 555 | StartNextDecodeForceKeyframe(); |
| 556 | // Initial keyframe. |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 557 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 558 | test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build())); |
| 559 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 560 | |
Evan Shrubsole | 6cd6d8e | 2022-02-11 14:30:26 | [diff] [blame] | 561 | time_controller_.AdvanceTime(kFps30Delay / 2); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 562 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| 563 | .Id(1) |
| 564 | .Time(kFps30Rtp) |
| 565 | .Refs({0}) |
| 566 | .AsLast() |
| 567 | .Build())); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 568 | StartNextDecode(); |
Florent Castelli | 8037fc6 | 2024-08-29 13:00:40 | [diff] [blame] | 569 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Eq(std::nullopt)); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 570 | |
| 571 | // Scheduler is waiting to deliver Frame 1 now. Insert Frame 2. Frame 1 should |
| 572 | // be delivered still. |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 573 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| 574 | .Id(2) |
| 575 | .Time(kFps30Rtp * 2) |
| 576 | .Refs({0}) |
| 577 | .AsLast() |
| 578 | .Build())); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 579 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 580 | } |
| 581 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 582 | TEST_P(VideoStreamBufferControllerTest, SameFrameNotScheduledTwice) { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 583 | // A frame could be scheduled twice if last_frame() arrive out-of-order but |
| 584 | // the older frame is old enough to be fast forwarded. |
| 585 | // |
| 586 | // 1. F2 arrives and is scheduled. |
| 587 | // 2. F3 arrives, but scheduling will not change since F2 is next. |
| 588 | // 3. F1 arrives late and scheduling is checked since it is before F2. F1 |
| 589 | // fast-forwarded since it is older. |
| 590 | // |
| 591 | // F2 is the best frame, but should only be scheduled once, followed by F3. |
| 592 | StartNextDecodeForceKeyframe(); |
| 593 | |
| 594 | // First keyframe. |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 595 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 596 | test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build())); |
| 597 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Millis(15)), |
| 598 | Frame(test::WithId(0))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 599 | |
| 600 | StartNextDecode(); |
| 601 | |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 602 | // F2 arrives and is scheduled. |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 603 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
Evan Shrubsole | 214cab5 | 2022-08-16 09:48:23 | [diff] [blame] | 604 | test::FakeFrameBuilder().Id(2).Time(2 * kFps30Rtp).AsLast().Build())); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 605 | |
| 606 | // F3 arrives before F2 is extracted. |
| 607 | time_controller_.AdvanceTime(kFps30Delay); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 608 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
Evan Shrubsole | 214cab5 | 2022-08-16 09:48:23 | [diff] [blame] | 609 | test::FakeFrameBuilder().Id(3).Time(3 * kFps30Rtp).AsLast().Build())); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 610 | |
| 611 | // F1 arrives and is fast-forwarded since it is too late. |
| 612 | // F2 is already scheduled and should not be rescheduled. |
| 613 | time_controller_.AdvanceTime(kFps30Delay / 2); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 614 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
Evan Shrubsole | 214cab5 | 2022-08-16 09:48:23 | [diff] [blame] | 615 | test::FakeFrameBuilder().Id(1).Time(1 * kFps30Rtp).AsLast().Build())); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 616 | |
Evan Shrubsole | 214cab5 | 2022-08-16 09:48:23 | [diff] [blame] | 617 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 618 | StartNextDecode(); |
| 619 | |
Evan Shrubsole | 214cab5 | 2022-08-16 09:48:23 | [diff] [blame] | 620 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(3))); |
Evan Shrubsole | 6cd6d8e | 2022-02-11 14:30:26 | [diff] [blame] | 621 | StartNextDecode(); |
| 622 | EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 623 | EXPECT_EQ(dropped_frames(), 1); |
| 624 | } |
| 625 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 626 | TEST_P(VideoStreamBufferControllerTest, TestStatsCallback) { |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 627 | EXPECT_CALL(stats_callback_, |
| 628 | OnCompleteFrame(true, kFrameSize, VideoContentType::UNSPECIFIED)); |
Rasmus Brandt | 621cb29 | 2023-05-24 11:29:14 | [diff] [blame] | 629 | EXPECT_CALL(stats_callback_, OnDecodableFrame); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 630 | EXPECT_CALL(stats_callback_, OnFrameBufferTimingsUpdated); |
| 631 | |
| 632 | // Fake timing having received decoded frame. |
Evan Shrubsole | d6cdf80 | 2022-03-02 14:13:55 | [diff] [blame] | 633 | timing_.StopDecodeTimer(TimeDelta::Millis(1), clock_->CurrentTime()); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 634 | StartNextDecodeForceKeyframe(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 635 | buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 636 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 637 | |
| 638 | // Flush stats posted on the decode queue. |
| 639 | time_controller_.AdvanceTime(TimeDelta::Zero()); |
| 640 | } |
| 641 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 642 | TEST_P(VideoStreamBufferControllerTest, |
| 643 | FrameCompleteCalledOnceForDuplicateFrame) { |
Evan Shrubsole | d4ff12f | 2022-03-01 09:16:11 | [diff] [blame] | 644 | EXPECT_CALL(stats_callback_, |
| 645 | OnCompleteFrame(true, kFrameSize, VideoContentType::UNSPECIFIED)) |
| 646 | .Times(1); |
| 647 | |
| 648 | StartNextDecodeForceKeyframe(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 649 | buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
| 650 | buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); |
Evan Shrubsole | d4ff12f | 2022-03-01 09:16:11 | [diff] [blame] | 651 | // Flush stats posted on the decode queue. |
| 652 | time_controller_.AdvanceTime(TimeDelta::Zero()); |
| 653 | } |
| 654 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 655 | TEST_P(VideoStreamBufferControllerTest, |
| 656 | FrameCompleteCalledOnceForSingleTemporalUnit) { |
Evan Shrubsole | d4ff12f | 2022-03-01 09:16:11 | [diff] [blame] | 657 | StartNextDecodeForceKeyframe(); |
| 658 | |
| 659 | // `OnCompleteFrame` should not be called for the first two frames since they |
| 660 | // do not complete the temporal layer. |
| 661 | EXPECT_CALL(stats_callback_, OnCompleteFrame(_, _, _)).Times(0); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 662 | buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).Build()); |
| 663 | buffer_->InsertFrame( |
| 664 | test::FakeFrameBuilder().Id(1).Time(0).Refs({0}).Build()); |
Evan Shrubsole | d4ff12f | 2022-03-01 09:16:11 | [diff] [blame] | 665 | time_controller_.AdvanceTime(TimeDelta::Zero()); |
| 666 | // Flush stats posted on the decode queue. |
| 667 | ::testing::Mock::VerifyAndClearExpectations(&stats_callback_); |
| 668 | |
| 669 | // Note that this frame is not marked as a keyframe since the last spatial |
| 670 | // layer has dependencies. |
| 671 | EXPECT_CALL(stats_callback_, |
| 672 | OnCompleteFrame(false, kFrameSize, VideoContentType::UNSPECIFIED)) |
| 673 | .Times(1); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 674 | buffer_->InsertFrame( |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 675 | test::FakeFrameBuilder().Id(2).Time(0).Refs({0, 1}).AsLast().Build()); |
Evan Shrubsole | d4ff12f | 2022-03-01 09:16:11 | [diff] [blame] | 676 | // Flush stats posted on the decode queue. |
| 677 | time_controller_.AdvanceTime(TimeDelta::Zero()); |
| 678 | } |
| 679 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 680 | TEST_P(VideoStreamBufferControllerTest, |
| 681 | FrameCompleteCalledOnceForCompleteTemporalUnit) { |
Evan Shrubsole | d4ff12f | 2022-03-01 09:16:11 | [diff] [blame] | 682 | // FrameBuffer2 logs the complete frame on the arrival of the last layer. |
Evan Shrubsole | d4ff12f | 2022-03-01 09:16:11 | [diff] [blame] | 683 | StartNextDecodeForceKeyframe(); |
| 684 | |
| 685 | // `OnCompleteFrame` should not be called for the first two frames since they |
| 686 | // do not complete the temporal layer. Frame 1 arrives later, at which time |
| 687 | // this frame can finally be considered complete. |
| 688 | EXPECT_CALL(stats_callback_, OnCompleteFrame(_, _, _)).Times(0); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 689 | buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).Build()); |
| 690 | buffer_->InsertFrame( |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 691 | test::FakeFrameBuilder().Id(2).Time(0).Refs({0, 1}).AsLast().Build()); |
Evan Shrubsole | d4ff12f | 2022-03-01 09:16:11 | [diff] [blame] | 692 | time_controller_.AdvanceTime(TimeDelta::Zero()); |
| 693 | // Flush stats posted on the decode queue. |
| 694 | ::testing::Mock::VerifyAndClearExpectations(&stats_callback_); |
| 695 | |
| 696 | EXPECT_CALL(stats_callback_, |
| 697 | OnCompleteFrame(false, kFrameSize, VideoContentType::UNSPECIFIED)) |
| 698 | .Times(1); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 699 | buffer_->InsertFrame( |
| 700 | test::FakeFrameBuilder().Id(1).Time(0).Refs({0}).Build()); |
Evan Shrubsole | d4ff12f | 2022-03-01 09:16:11 | [diff] [blame] | 701 | // Flush stats posted on the decode queue. |
| 702 | time_controller_.AdvanceTime(TimeDelta::Zero()); |
| 703 | } |
| 704 | |
Evan Shrubsole | 6cd6d8e | 2022-02-11 14:30:26 | [diff] [blame] | 705 | // Note: This test takes a long time to run if the fake metronome is active. |
| 706 | // Since the test needs to wait for the timestamp to rollover, it has a fake |
| 707 | // delay of around 6.5 hours. Even though time is simulated, this will be |
| 708 | // around 1,500,000 metronome tick invocations. |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 709 | TEST_P(VideoStreamBufferControllerTest, NextFrameWithOldTimestamp) { |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 710 | // Test inserting 31 frames and pause the stream for a long time before |
| 711 | // frame 32. |
| 712 | StartNextDecodeForceKeyframe(); |
| 713 | constexpr uint32_t kBaseRtp = std::numeric_limits<uint32_t>::max() / 2; |
| 714 | |
| 715 | // First keyframe. The receive time must be explicitly set in this test since |
| 716 | // the RTP derived time used in all tests does not work when the long pause |
| 717 | // happens later in the test. |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 718 | buffer_->InsertFrame(test::FakeFrameBuilder() |
| 719 | .Id(0) |
| 720 | .Time(kBaseRtp) |
| 721 | .ReceivedTime(clock_->CurrentTime()) |
| 722 | .AsLast() |
| 723 | .Build()); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 724 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(0))); |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 725 | |
| 726 | // 1 more frame to warmup VCMTiming for 30fps. |
| 727 | StartNextDecode(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 728 | buffer_->InsertFrame(test::FakeFrameBuilder() |
| 729 | .Id(1) |
| 730 | .Time(kBaseRtp + kFps30Rtp) |
| 731 | .ReceivedTime(clock_->CurrentTime()) |
| 732 | .AsLast() |
| 733 | .Build()); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 734 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 735 | |
| 736 | // Pause the stream for such a long time it incurs an RTP timestamp rollover |
| 737 | // by over half. |
| 738 | constexpr uint32_t kLastRtp = kBaseRtp + kFps30Rtp; |
| 739 | constexpr uint32_t kRolloverRtp = |
| 740 | kLastRtp + std::numeric_limits<uint32_t>::max() / 2 + 1; |
| 741 | constexpr Frequency kRtpHz = Frequency::KiloHertz(90); |
| 742 | // Pause for corresponding delay such that RTP timestamp would increase this |
| 743 | // much at 30fps. |
| 744 | constexpr TimeDelta kRolloverDelay = |
| 745 | (std::numeric_limits<uint32_t>::max() / 2 + 1) / kRtpHz; |
| 746 | |
| 747 | // Avoid timeout being set while waiting for the frame and before the receiver |
| 748 | // is ready. |
| 749 | ResetLastResult(); |
Florent Castelli | 8037fc6 | 2024-08-29 13:00:40 | [diff] [blame] | 750 | EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), Eq(std::nullopt)); |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 751 | time_controller_.AdvanceTime(kRolloverDelay - kMaxWaitForFrame); |
| 752 | StartNextDecode(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 753 | buffer_->InsertFrame(test::FakeFrameBuilder() |
| 754 | .Id(2) |
| 755 | .Time(kRolloverRtp) |
| 756 | .ReceivedTime(clock_->CurrentTime()) |
| 757 | .AsLast() |
| 758 | .Build()); |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 759 | // FrameBuffer2 drops the frame, while FrameBuffer3 will continue the stream. |
Evan Shrubsole | 4c26932 | 2022-08-02 08:57:04 | [diff] [blame] | 760 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2))); |
Evan Shrubsole | 9a99905 | 2021-12-12 14:27:00 | [diff] [blame] | 761 | } |
| 762 | |
Evan Shrubsole | 8fe5579 | 2022-10-27 11:05:44 | [diff] [blame] | 763 | TEST_P(VideoStreamBufferControllerTest, |
| 764 | FrameNotSetForDecodedIfFrameBufferBecomesNonDecodable) { |
| 765 | // This can happen if the frame buffer receives non-standard input. This test |
| 766 | // will simply clear the frame buffer to replicate this. |
| 767 | StartNextDecodeForceKeyframe(); |
| 768 | // Initial keyframe. |
| 769 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| 770 | test::FakeFrameBuilder().Id(0).Time(0).SpatialLayer(1).AsLast().Build())); |
| 771 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
| 772 | |
| 773 | // Insert a frame that will become non-decodable. |
| 774 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| 775 | .Id(11) |
| 776 | .Time(kFps30Rtp) |
| 777 | .Refs({0}) |
| 778 | .SpatialLayer(1) |
| 779 | .AsLast() |
| 780 | .Build())); |
| 781 | StartNextDecode(); |
| 782 | // Second layer inserted after last layer for the same frame out-of-order. |
| 783 | // This second frame requires some older frame to be decoded and so now the |
| 784 | // super-frame is no longer decodable despite already being scheduled. |
| 785 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| 786 | .Id(10) |
| 787 | .Time(kFps30Rtp) |
| 788 | .SpatialLayer(0) |
| 789 | .Refs({2}) |
| 790 | .Build())); |
| 791 | EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); |
| 792 | |
| 793 | // Ensure that this frame can be decoded later. |
| 794 | StartNextDecode(); |
| 795 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() |
| 796 | .Id(2) |
| 797 | .Time(kFps30Rtp / 2) |
| 798 | .SpatialLayer(0) |
| 799 | .Refs({0}) |
| 800 | .AsLast() |
| 801 | .Build())); |
| 802 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2))); |
| 803 | StartNextDecode(); |
| 804 | EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(10))); |
| 805 | } |
| 806 | |
Evan Shrubsole | a006ba1 | 2022-09-05 10:09:08 | [diff] [blame] | 807 | INSTANTIATE_TEST_SUITE_P(VideoStreamBufferController, |
| 808 | VideoStreamBufferControllerTest, |
| 809 | ::testing::Combine(::testing::Bool(), |
| 810 | ::testing::Values("")), |
| 811 | [](const auto& info) { |
| 812 | return std::get<0>(info.param) ? "SyncDecoding" |
| 813 | : "UnsyncedDecoding"; |
| 814 | }); |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 815 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 816 | class LowLatencyVideoStreamBufferControllerTest |
| 817 | : public ::testing::Test, |
| 818 | public VideoStreamBufferControllerFixture {}; |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 819 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 820 | TEST_P(LowLatencyVideoStreamBufferControllerTest, |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 821 | FramesDecodedInstantlyWithLowLatencyRendering) { |
| 822 | // Initial keyframe. |
| 823 | StartNextDecodeForceKeyframe(); |
Evan Shrubsole | d6cdf80 | 2022-03-02 14:13:55 | [diff] [blame] | 824 | timing_.set_min_playout_delay(TimeDelta::Zero()); |
| 825 | timing_.set_max_playout_delay(TimeDelta::Millis(10)); |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 826 | // Playout delay of 0 implies low-latency rendering. |
Danil Chapovalov | c146b5f | 2023-08-16 09:42:52 | [diff] [blame] | 827 | auto frame = test::FakeFrameBuilder() |
| 828 | .Id(0) |
| 829 | .Time(0) |
Danil Chapovalov | 7084e1b | 2023-08-23 11:16:22 | [diff] [blame] | 830 | .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(10)}) |
Danil Chapovalov | c146b5f | 2023-08-16 09:42:52 | [diff] [blame] | 831 | .AsLast() |
| 832 | .Build(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 833 | buffer_->InsertFrame(std::move(frame)); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 834 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 835 | |
| 836 | // Delta frame would normally wait here, but should decode at the pacing rate |
| 837 | // in low-latency mode. |
| 838 | StartNextDecode(); |
Danil Chapovalov | c146b5f | 2023-08-16 09:42:52 | [diff] [blame] | 839 | frame = test::FakeFrameBuilder() |
| 840 | .Id(1) |
| 841 | .Time(kFps30Rtp) |
Danil Chapovalov | 7084e1b | 2023-08-23 11:16:22 | [diff] [blame] | 842 | .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(10)}) |
Danil Chapovalov | c146b5f | 2023-08-16 09:42:52 | [diff] [blame] | 843 | .AsLast() |
| 844 | .Build(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 845 | buffer_->InsertFrame(std::move(frame)); |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 846 | // Pacing is set to 16ms in the field trial so we should not decode yet. |
Florent Castelli | 8037fc6 | 2024-08-29 13:00:40 | [diff] [blame] | 847 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Eq(std::nullopt)); |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 848 | time_controller_.AdvanceTime(TimeDelta::Millis(16)); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 849 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1))); |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 850 | } |
| 851 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 852 | TEST_P(LowLatencyVideoStreamBufferControllerTest, ZeroPlayoutDelayFullQueue) { |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 853 | // Initial keyframe. |
| 854 | StartNextDecodeForceKeyframe(); |
Evan Shrubsole | d6cdf80 | 2022-03-02 14:13:55 | [diff] [blame] | 855 | timing_.set_min_playout_delay(TimeDelta::Zero()); |
| 856 | timing_.set_max_playout_delay(TimeDelta::Millis(10)); |
Danil Chapovalov | c146b5f | 2023-08-16 09:42:52 | [diff] [blame] | 857 | auto frame = test::FakeFrameBuilder() |
| 858 | .Id(0) |
| 859 | .Time(0) |
Danil Chapovalov | 7084e1b | 2023-08-23 11:16:22 | [diff] [blame] | 860 | .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(10)}) |
Danil Chapovalov | c146b5f | 2023-08-16 09:42:52 | [diff] [blame] | 861 | .AsLast() |
| 862 | .Build(); |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 863 | // Playout delay of 0 implies low-latency rendering. |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 864 | buffer_->InsertFrame(std::move(frame)); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 865 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 866 | |
| 867 | // Queue up 5 frames (configured max queue size for 0-playout delay pacing). |
| 868 | for (int id = 1; id <= 6; ++id) { |
Danil Chapovalov | c146b5f | 2023-08-16 09:42:52 | [diff] [blame] | 869 | frame = test::FakeFrameBuilder() |
| 870 | .Id(id) |
| 871 | .Time(kFps30Rtp * id) |
Danil Chapovalov | 7084e1b | 2023-08-23 11:16:22 | [diff] [blame] | 872 | .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(10)}) |
Danil Chapovalov | c146b5f | 2023-08-16 09:42:52 | [diff] [blame] | 873 | .AsLast() |
| 874 | .Build(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 875 | buffer_->InsertFrame(std::move(frame)); |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 876 | } |
| 877 | |
| 878 | // The queue is at its max size for zero playout delay pacing, so the pacing |
| 879 | // should be ignored and the next frame should be decoded instantly. |
| 880 | StartNextDecode(); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 881 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1))); |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 882 | } |
| 883 | |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 884 | TEST_P(LowLatencyVideoStreamBufferControllerTest, |
| 885 | MinMaxDelayZeroLowLatencyMode) { |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 886 | // Initial keyframe. |
| 887 | StartNextDecodeForceKeyframe(); |
Evan Shrubsole | d6cdf80 | 2022-03-02 14:13:55 | [diff] [blame] | 888 | timing_.set_min_playout_delay(TimeDelta::Zero()); |
| 889 | timing_.set_max_playout_delay(TimeDelta::Zero()); |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 890 | // Playout delay of 0 implies low-latency rendering. |
Danil Chapovalov | c146b5f | 2023-08-16 09:42:52 | [diff] [blame] | 891 | auto frame = test::FakeFrameBuilder() |
| 892 | .Id(0) |
| 893 | .Time(0) |
| 894 | .PlayoutDelay(VideoPlayoutDelay::Minimal()) |
| 895 | .AsLast() |
| 896 | .Build(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 897 | buffer_->InsertFrame(std::move(frame)); |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 898 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 899 | |
| 900 | // Delta frame would normally wait here, but should decode at the pacing rate |
| 901 | // in low-latency mode. |
| 902 | StartNextDecode(); |
Danil Chapovalov | c146b5f | 2023-08-16 09:42:52 | [diff] [blame] | 903 | frame = test::FakeFrameBuilder() |
| 904 | .Id(1) |
| 905 | .Time(kFps30Rtp) |
| 906 | .PlayoutDelay(VideoPlayoutDelay::Minimal()) |
| 907 | .AsLast() |
| 908 | .Build(); |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 909 | buffer_->InsertFrame(std::move(frame)); |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 910 | // The min/max=0 version of low-latency rendering will result in a large |
| 911 | // negative decode wait time, so the frame should be ready right away. |
Evan Shrubsole | a0ee64c | 2022-04-26 08:09:04 | [diff] [blame] | 912 | EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1))); |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 913 | } |
| 914 | |
| 915 | INSTANTIATE_TEST_SUITE_P( |
Evan Shrubsole | 476f18d2 | 2022-08-15 15:21:16 | [diff] [blame] | 916 | VideoStreamBufferController, |
| 917 | LowLatencyVideoStreamBufferControllerTest, |
Evan Shrubsole | a006ba1 | 2022-09-05 10:09:08 | [diff] [blame] | 918 | ::testing::Combine( |
| 919 | ::testing::Bool(), |
| 920 | ::testing::Values( |
| 921 | "WebRTC-ZeroPlayoutDelay/min_pacing:16ms,max_decode_queue_size:5/", |
| 922 | "WebRTC-ZeroPlayoutDelay/" |
| 923 | "min_pacing:16ms,max_decode_queue_size:5/"))); |
Evan Shrubsole | f7a1937 | 2022-02-14 13:05:10 | [diff] [blame] | 924 | |
philipel | 7446b60 | 2022-10-06 13:49:17 | [diff] [blame] | 925 | class IncomingTimestampVideoStreamBufferControllerTest |
| 926 | : public ::testing::Test, |
| 927 | public VideoStreamBufferControllerFixture {}; |
| 928 | |
| 929 | TEST_P(IncomingTimestampVideoStreamBufferControllerTest, |
| 930 | IncomingTimestampOnMarkerBitOnly) { |
| 931 | StartNextDecodeForceKeyframe(); |
| 932 | EXPECT_CALL(timing_, IncomingTimestampMocked) |
| 933 | .Times(field_trials_.IsDisabled("WebRTC-IncomingTimestampOnMarkerBitOnly") |
| 934 | ? 3 |
| 935 | : 1); |
| 936 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| 937 | test::FakeFrameBuilder().Id(0).SpatialLayer(0).Time(0).Build())); |
| 938 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| 939 | test::FakeFrameBuilder().Id(1).SpatialLayer(1).Time(0).Build())); |
| 940 | buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( |
| 941 | test::FakeFrameBuilder().Id(2).SpatialLayer(2).Time(0).AsLast().Build())); |
| 942 | } |
| 943 | |
| 944 | INSTANTIATE_TEST_SUITE_P( |
| 945 | VideoStreamBufferController, |
| 946 | IncomingTimestampVideoStreamBufferControllerTest, |
| 947 | ::testing::Combine( |
| 948 | ::testing::Bool(), |
| 949 | ::testing::Values( |
| 950 | "WebRTC-IncomingTimestampOnMarkerBitOnly/Enabled/", |
| 951 | "WebRTC-IncomingTimestampOnMarkerBitOnly/Disabled/"))); |
| 952 | |
Evan Shrubsole | 9d29026 | 2021-12-15 13:33:40 | [diff] [blame] | 953 | } // namespace webrtc |