blob: c97601b944062072f7f04d5b00988dac7cfb3832 [file] [log] [blame]
Evan Shrubsole9d290262021-12-15 13:33:401/*
2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Evan Shrubsole476f18d22022-08-15 15:21:1611#include "video/video_stream_buffer_controller.h"
Evan Shrubsole9d290262021-12-15 13:33:4012
13#include <stdint.h>
14
Evan Shrubsole9a999052021-12-12 14:27:0015#include <limits>
Evan Shrubsole9d290262021-12-15 13:33:4016#include <memory>
Florent Castelli8037fc62024-08-29 13:00:4017#include <optional>
Evan Shrubsole9d290262021-12-15 13:33:4018#include <string>
Evan Shrubsolea006ba12022-09-05 10:09:0819#include <tuple>
Evan Shrubsole9d290262021-12-15 13:33:4020#include <utility>
21#include <vector>
22
Evan Shrubsole9a999052021-12-12 14:27:0023#include "absl/types/variant.h"
Evan Shrubsole6cd6d8e2022-02-11 14:30:2624#include "api/metronome/test/fake_metronome.h"
Evan Shrubsole9a999052021-12-12 14:27:0025#include "api/units/frequency.h"
Evan Shrubsole9d290262021-12-15 13:33:4026#include "api/units/time_delta.h"
27#include "api/units/timestamp.h"
28#include "api/video/video_content_type.h"
Evan Shrubsolef7a19372022-02-14 13:05:1029#include "api/video/video_timing.h"
Evan Shrubsole9a999052021-12-12 14:27:0030#include "rtc_base/checks.h"
Evan Shrubsolea0ee64c2022-04-26 08:09:0431#include "test/fake_encoded_frame.h"
Evan Shrubsole9d290262021-12-15 13:33:4032#include "test/gmock.h"
33#include "test/gtest.h"
Jonas Oreland8ca06132022-03-14 11:52:4834#include "test/scoped_key_value_config.h"
Evan Shrubsole9d290262021-12-15 13:33:4035#include "test/time_controller/simulated_time_controller.h"
Evan Shrubsole6cd6d8e2022-02-11 14:30:2636#include "video/decode_synchronizer.h"
Evan Shrubsolea006ba12022-09-05 10:09:0837#include "video/task_queue_frame_decode_scheduler.h"
Evan Shrubsole9d290262021-12-15 13:33:4038
Evan Shrubsole9a999052021-12-12 14:27:0039using ::testing::_;
Evan Shrubsole9d290262021-12-15 13:33:4040using ::testing::AllOf;
41using ::testing::Contains;
42using ::testing::Each;
43using ::testing::Eq;
44using ::testing::IsEmpty;
45using ::testing::Matches;
46using ::testing::Ne;
47using ::testing::Not;
48using ::testing::Optional;
49using ::testing::Pointee;
50using ::testing::SizeIs;
Evan Shrubsole9a999052021-12-12 14:27:0051using ::testing::VariantWith;
Evan Shrubsole9d290262021-12-15 13:33:4052
53namespace webrtc {
54
Evan Shrubsole9d290262021-12-15 13:33:4055namespace {
56
57constexpr size_t kFrameSize = 10;
58constexpr uint32_t kFps30Rtp = 90000 / 30;
59constexpr TimeDelta kFps30Delay = 1 / Frequency::Hertz(30);
Evan Shrubsole9d290262021-12-15 13:33:4060constexpr Timestamp kClockStart = Timestamp::Millis(1000);
61
Evan Shrubsole9a999052021-12-12 14:27:0062auto TimedOut() {
63 return Optional(VariantWith<TimeDelta>(_));
64}
65
66auto Frame(testing::Matcher<EncodedFrame> m) {
67 return Optional(VariantWith<std::unique_ptr<EncodedFrame>>(Pointee(m)));
68}
69
Evan Shrubsolea0ee64c2022-04-26 08:09:0470std::unique_ptr<test::FakeEncodedFrame> WithReceiveTimeFromRtpTimestamp(
71 std::unique_ptr<test::FakeEncodedFrame> frame) {
Danil Chapovalov9c584832023-09-18 13:48:4972 if (frame->RtpTimestamp() == 0) {
Evan Shrubsolea0ee64c2022-04-26 08:09:0473 frame->SetReceivedTime(kClockStart.ms());
74 } else {
75 frame->SetReceivedTime(
Danil Chapovalov9c584832023-09-18 13:48:4976 TimeDelta::Seconds(frame->RtpTimestamp() / 90000.0).ms() +
Evan Shrubsolea0ee64c2022-04-26 08:09:0477 kClockStart.ms());
Evan Shrubsole9d290262021-12-15 13:33:4078 }
Evan Shrubsolea0ee64c2022-04-26 08:09:0479 return frame;
80}
Evan Shrubsole9d290262021-12-15 13:33:4081
philipel7446b602022-10-06 13:49:1782class VCMTimingTest : public VCMTiming {
83 public:
84 using VCMTiming::VCMTiming;
85 void IncomingTimestamp(uint32_t rtp_timestamp,
86 Timestamp last_packet_time) override {
87 IncomingTimestampMocked(rtp_timestamp, last_packet_time);
88 VCMTiming::IncomingTimestamp(rtp_timestamp, last_packet_time);
89 }
90
91 MOCK_METHOD(void,
92 IncomingTimestampMocked,
93 (uint32_t rtp_timestamp, Timestamp last_packet_time),
94 ());
95};
96
Rasmus Brandt39250a42023-05-09 12:33:4397class VideoStreamBufferControllerStatsObserverMock
98 : public VideoStreamBufferControllerStatsObserver {
Evan Shrubsole9d290262021-12-15 13:33:4099 public:
100 MOCK_METHOD(void,
101 OnCompleteFrame,
102 (bool is_keyframe,
103 size_t size_bytes,
104 VideoContentType content_type),
105 (override));
106 MOCK_METHOD(void, OnDroppedFrames, (uint32_t num_dropped), (override));
107 MOCK_METHOD(void,
Rasmus Brandt621cb292023-05-24 11:29:14108 OnDecodableFrame,
Rasmus Brandtf0820ff2023-05-25 07:37:16109 (TimeDelta jitter_buffer_delay,
110 TimeDelta target_delay,
111 TimeDelta minimum_delay),
Rasmus Brandt621cb292023-05-24 11:29:14112 (override));
113 MOCK_METHOD(void,
Evan Shrubsole9d290262021-12-15 13:33:40114 OnFrameBufferTimingsUpdated,
Rasmus Brandt621cb292023-05-24 11:29:14115 (int estimated_max_decode_time_ms,
Evan Shrubsole9d290262021-12-15 13:33:40116 int current_delay_ms,
117 int target_delay_ms,
Rasmus Brandt621cb292023-05-24 11:29:14118 int jitter_delay_ms,
Evan Shrubsole9d290262021-12-15 13:33:40119 int min_playout_delay_ms,
120 int render_delay_ms),
121 (override));
122 MOCK_METHOD(void,
123 OnTimingFrameInfoUpdated,
124 (const TimingFrameInfo& info),
125 (override));
126};
Evan Shrubsoled4ff12f2022-03-01 09:16:11127
Evan Shrubsole9d290262021-12-15 13:33:40128} // namespace
129
130constexpr auto kMaxWaitForKeyframe = TimeDelta::Millis(500);
131constexpr auto kMaxWaitForFrame = TimeDelta::Millis(1500);
Evan Shrubsole476f18d22022-08-15 15:21:16132class VideoStreamBufferControllerFixture
Evan Shrubsolea006ba12022-09-05 10:09:08133 : public ::testing::WithParamInterface<std::tuple<bool, std::string>>,
Evan Shrubsolef7a19372022-02-14 13:05:10134 public FrameSchedulingReceiver {
Evan Shrubsole9d290262021-12-15 13:33:40135 public:
Evan Shrubsole476f18d22022-08-15 15:21:16136 VideoStreamBufferControllerFixture()
Evan Shrubsolea006ba12022-09-05 10:09:08137 : sync_decoding_(std::get<0>(GetParam())),
138 field_trials_(std::get<1>(GetParam())),
Evan Shrubsole9d290262021-12-15 13:33:40139 time_controller_(kClockStart),
140 clock_(time_controller_.GetClock()),
Markus Handellbe400e42022-11-08 11:14:23141 fake_metronome_(TimeDelta::Millis(16)),
Evan Shrubsole214cab52022-08-16 09:48:23142 decode_sync_(clock_,
143 &fake_metronome_,
144 time_controller_.GetMainThread()),
Jonas Orelande02f9ee2022-03-25 11:43:14145 timing_(clock_, field_trials_),
Evan Shrubsolea006ba12022-09-05 10:09:08146 buffer_(std::make_unique<VideoStreamBufferController>(
Evan Shrubsole476f18d22022-08-15 15:21:16147 clock_,
Evan Shrubsole214cab52022-08-16 09:48:23148 time_controller_.GetMainThread(),
Evan Shrubsole476f18d22022-08-15 15:21:16149 &timing_,
150 &stats_callback_,
Evan Shrubsole476f18d22022-08-15 15:21:16151 this,
152 kMaxWaitForKeyframe,
153 kMaxWaitForFrame,
Evan Shrubsolea006ba12022-09-05 10:09:08154 sync_decoding_ ? decode_sync_.CreateSynchronizedFrameScheduler()
155 : std::make_unique<TaskQueueFrameDecodeScheduler>(
156 clock_,
157 time_controller_.GetMainThread()),
Evan Shrubsole476f18d22022-08-15 15:21:16158 field_trials_)) {
Evan Shrubsole9d290262021-12-15 13:33:40159 // Avoid starting with negative render times.
Evan Shrubsoled6cdf802022-03-02 14:13:55160 timing_.set_min_playout_delay(TimeDelta::Millis(10));
Evan Shrubsole9d290262021-12-15 13:33:40161
162 ON_CALL(stats_callback_, OnDroppedFrames)
163 .WillByDefault(
164 [this](auto num_dropped) { dropped_frames_ += num_dropped; });
165 }
166
Evan Shrubsole476f18d22022-08-15 15:21:16167 ~VideoStreamBufferControllerFixture() override {
168 if (buffer_) {
Evan Shrubsole214cab52022-08-16 09:48:23169 buffer_->Stop();
Evan Shrubsole9d290262021-12-15 13:33:40170 }
Evan Shrubsole6cd6d8e2022-02-11 14:30:26171 time_controller_.AdvanceTime(TimeDelta::Zero());
Evan Shrubsole9d290262021-12-15 13:33:40172 }
173
174 void OnEncodedFrame(std::unique_ptr<EncodedFrame> frame) override {
Evan Shrubsole9a999052021-12-12 14:27:00175 RTC_DCHECK(frame);
176 SetWaitResult(std::move(frame));
Evan Shrubsole9d290262021-12-15 13:33:40177 }
178
179 void OnDecodableFrameTimeout(TimeDelta wait_time) override {
Evan Shrubsole9a999052021-12-12 14:27:00180 SetWaitResult(wait_time);
Evan Shrubsole9d290262021-12-15 13:33:40181 }
182
Evan Shrubsole9a999052021-12-12 14:27:00183 using WaitResult =
184 absl::variant<std::unique_ptr<EncodedFrame>, TimeDelta /*wait_time*/>;
185
Florent Castelli8037fc62024-08-29 13:00:40186 std::optional<WaitResult> WaitForFrameOrTimeout(TimeDelta wait) {
Evan Shrubsole9a999052021-12-12 14:27:00187 if (wait_result_) {
188 return std::move(wait_result_);
Evan Shrubsole9d290262021-12-15 13:33:40189 }
Evan Shrubsole214cab52022-08-16 09:48:23190 time_controller_.AdvanceTime(TimeDelta::Zero());
191 if (wait_result_) {
192 return std::move(wait_result_);
193 }
Evan Shrubsole9a999052021-12-12 14:27:00194
Evan Shrubsole214cab52022-08-16 09:48:23195 Timestamp now = clock_->CurrentTime();
196 // TODO(bugs.webrtc.org/13756): Remove this when rtc::Thread uses uses
197 // Timestamp instead of an integer milliseconds. This extra wait is needed
198 // for some tests that use the metronome. This is due to rounding
199 // milliseconds, affecting the precision of simulated time controller uses
200 // when posting tasks from threads.
201 TimeDelta potential_extra_wait =
202 Timestamp::Millis((now + wait).ms()) - (now + wait);
Evan Shrubsole9d290262021-12-15 13:33:40203
Evan Shrubsole214cab52022-08-16 09:48:23204 time_controller_.AdvanceTime(wait);
205 if (potential_extra_wait > TimeDelta::Zero()) {
206 time_controller_.AdvanceTime(potential_extra_wait);
207 }
Evan Shrubsole9a999052021-12-12 14:27:00208 return std::move(wait_result_);
Evan Shrubsole9d290262021-12-15 13:33:40209 }
210
211 void StartNextDecode() {
212 ResetLastResult();
Evan Shrubsole476f18d22022-08-15 15:21:16213 buffer_->StartNextDecode(false);
Evan Shrubsole9d290262021-12-15 13:33:40214 }
215
216 void StartNextDecodeForceKeyframe() {
217 ResetLastResult();
Evan Shrubsole476f18d22022-08-15 15:21:16218 buffer_->StartNextDecode(true);
Evan Shrubsole9d290262021-12-15 13:33:40219 }
220
Evan Shrubsole9a999052021-12-12 14:27:00221 void ResetLastResult() { wait_result_.reset(); }
Evan Shrubsole9d290262021-12-15 13:33:40222
Evan Shrubsole9d290262021-12-15 13:33:40223 int dropped_frames() const { return dropped_frames_; }
224
225 protected:
Evan Shrubsolea006ba12022-09-05 10:09:08226 const bool sync_decoding_;
Jonas Oreland8ca06132022-03-14 11:52:48227 test::ScopedKeyValueConfig field_trials_;
Evan Shrubsole9d290262021-12-15 13:33:40228 GlobalSimulatedTimeController time_controller_;
229 Clock* const clock_;
Evan Shrubsole6cd6d8e2022-02-11 14:30:26230 test::FakeMetronome fake_metronome_;
231 DecodeSynchronizer decode_sync_;
Evan Shrubsole6cd6d8e2022-02-11 14:30:26232
philipel7446b602022-10-06 13:49:17233 ::testing::NiceMock<VCMTimingTest> timing_;
Rasmus Brandt39250a42023-05-09 12:33:43234 ::testing::NiceMock<VideoStreamBufferControllerStatsObserverMock>
235 stats_callback_;
Evan Shrubsole476f18d22022-08-15 15:21:16236 std::unique_ptr<VideoStreamBufferController> buffer_;
Evan Shrubsole9d290262021-12-15 13:33:40237
238 private:
Evan Shrubsole9a999052021-12-12 14:27:00239 void SetWaitResult(WaitResult result) {
240 RTC_DCHECK(!wait_result_);
241 if (absl::holds_alternative<std::unique_ptr<EncodedFrame>>(result)) {
242 RTC_DCHECK(absl::get<std::unique_ptr<EncodedFrame>>(result));
243 }
244 wait_result_.emplace(std::move(result));
Evan Shrubsole9d290262021-12-15 13:33:40245 }
246
Evan Shrubsole9d290262021-12-15 13:33:40247 uint32_t dropped_frames_ = 0;
Florent Castelli8037fc62024-08-29 13:00:40248 std::optional<WaitResult> wait_result_;
Evan Shrubsole9d290262021-12-15 13:33:40249};
250
Evan Shrubsole476f18d22022-08-15 15:21:16251class VideoStreamBufferControllerTest
252 : public ::testing::Test,
253 public VideoStreamBufferControllerFixture {};
Evan Shrubsolef7a19372022-02-14 13:05:10254
Evan Shrubsole476f18d22022-08-15 15:21:16255TEST_P(VideoStreamBufferControllerTest,
256 InitialTimeoutAfterKeyframeTimeoutPeriod) {
Evan Shrubsole9d290262021-12-15 13:33:40257 StartNextDecodeForceKeyframe();
Evan Shrubsole4c269322022-08-02 08:57:04258 // No frame inserted. Timeout expected.
Evan Shrubsole9a999052021-12-12 14:27:00259 EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), TimedOut());
Evan Shrubsole9d290262021-12-15 13:33:40260
261 // No new timeout set since receiver has not started new decode.
262 ResetLastResult();
Florent Castelli8037fc62024-08-29 13:00:40263 EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), Eq(std::nullopt));
Evan Shrubsole9d290262021-12-15 13:33:40264
265 // Now that receiver has asked for new frame, a new timeout can occur.
266 StartNextDecodeForceKeyframe();
Evan Shrubsole9a999052021-12-12 14:27:00267 EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), TimedOut());
Evan Shrubsole9d290262021-12-15 13:33:40268}
269
Evan Shrubsole476f18d22022-08-15 15:21:16270TEST_P(VideoStreamBufferControllerTest, KeyFramesAreScheduled) {
Evan Shrubsole9d290262021-12-15 13:33:40271 StartNextDecodeForceKeyframe();
272 time_controller_.AdvanceTime(TimeDelta::Millis(50));
273
Evan Shrubsolea0ee64c2022-04-26 08:09:04274 auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build();
Evan Shrubsole476f18d22022-08-15 15:21:16275 buffer_->InsertFrame(std::move(frame));
Evan Shrubsole9d290262021-12-15 13:33:40276
Evan Shrubsolea0ee64c2022-04-26 08:09:04277 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
Evan Shrubsole9d290262021-12-15 13:33:40278}
279
Evan Shrubsole476f18d22022-08-15 15:21:16280TEST_P(VideoStreamBufferControllerTest,
281 DeltaFrameTimeoutAfterKeyframeExtracted) {
Evan Shrubsole9d290262021-12-15 13:33:40282 StartNextDecodeForceKeyframe();
283
284 time_controller_.AdvanceTime(TimeDelta::Millis(50));
Evan Shrubsolea0ee64c2022-04-26 08:09:04285 auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build();
Evan Shrubsole476f18d22022-08-15 15:21:16286 buffer_->InsertFrame(std::move(frame));
Evan Shrubsolea0ee64c2022-04-26 08:09:04287 EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe),
288 Frame(test::WithId(0)));
Evan Shrubsole9d290262021-12-15 13:33:40289
290 StartNextDecode();
291 time_controller_.AdvanceTime(TimeDelta::Millis(50));
292
293 // Timeouts should now happen at the normal frequency.
294 const int expected_timeouts = 5;
295 for (int i = 0; i < expected_timeouts; ++i) {
Evan Shrubsole9a999052021-12-12 14:27:00296 EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut());
Evan Shrubsole9d290262021-12-15 13:33:40297 StartNextDecode();
298 }
Evan Shrubsole9d290262021-12-15 13:33:40299}
300
Evan Shrubsole476f18d22022-08-15 15:21:16301TEST_P(VideoStreamBufferControllerTest, DependantFramesAreScheduled) {
Evan Shrubsole9d290262021-12-15 13:33:40302 StartNextDecodeForceKeyframe();
Evan Shrubsole476f18d22022-08-15 15:21:16303 buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
Evan Shrubsolea0ee64c2022-04-26 08:09:04304 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
Evan Shrubsole9d290262021-12-15 13:33:40305
306 StartNextDecode();
307
308 time_controller_.AdvanceTime(kFps30Delay);
Evan Shrubsole476f18d22022-08-15 15:21:16309 buffer_->InsertFrame(test::FakeFrameBuilder()
310 .Id(1)
311 .Time(kFps30Rtp)
312 .AsLast()
313 .Refs({0})
314 .Build());
Evan Shrubsolea0ee64c2022-04-26 08:09:04315 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1)));
Evan Shrubsole9d290262021-12-15 13:33:40316}
317
Evan Shrubsole476f18d22022-08-15 15:21:16318TEST_P(VideoStreamBufferControllerTest, SpatialLayersAreScheduled) {
Evan Shrubsole9d290262021-12-15 13:33:40319 StartNextDecodeForceKeyframe();
Evan Shrubsole476f18d22022-08-15 15:21:16320 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
Evan Shrubsolea0ee64c2022-04-26 08:09:04321 test::FakeFrameBuilder().Id(0).SpatialLayer(0).Time(0).Build()));
Evan Shrubsole476f18d22022-08-15 15:21:16322 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
Evan Shrubsolea0ee64c2022-04-26 08:09:04323 test::FakeFrameBuilder().Id(1).SpatialLayer(1).Time(0).Build()));
Evan Shrubsole476f18d22022-08-15 15:21:16324 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
Evan Shrubsolea0ee64c2022-04-26 08:09:04325 test::FakeFrameBuilder().Id(2).SpatialLayer(2).Time(0).AsLast().Build()));
326 EXPECT_THAT(
327 WaitForFrameOrTimeout(TimeDelta::Zero()),
328 Frame(AllOf(test::WithId(0), test::FrameWithSize(3 * kFrameSize))));
Evan Shrubsole9d290262021-12-15 13:33:40329
Evan Shrubsole476f18d22022-08-15 15:21:16330 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
Evan Shrubsolea0ee64c2022-04-26 08:09:04331 test::FakeFrameBuilder().Id(3).Time(kFps30Rtp).SpatialLayer(0).Build()));
Evan Shrubsole476f18d22022-08-15 15:21:16332 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
Evan Shrubsolea0ee64c2022-04-26 08:09:04333 test::FakeFrameBuilder().Id(4).Time(kFps30Rtp).SpatialLayer(1).Build()));
Evan Shrubsole476f18d22022-08-15 15:21:16334 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder()
335 .Id(5)
336 .Time(kFps30Rtp)
337 .SpatialLayer(2)
338 .AsLast()
339 .Build()));
Evan Shrubsole9d290262021-12-15 13:33:40340
341 StartNextDecode();
Evan Shrubsolea0ee64c2022-04-26 08:09:04342 EXPECT_THAT(
343 WaitForFrameOrTimeout(kFps30Delay * 10),
344 Frame(AllOf(test::WithId(3), test::FrameWithSize(3 * kFrameSize))));
Evan Shrubsole9d290262021-12-15 13:33:40345}
346
Evan Shrubsole476f18d22022-08-15 15:21:16347TEST_P(VideoStreamBufferControllerTest,
348 OutstandingFrameTasksAreCancelledAfterDeletion) {
Evan Shrubsole9d290262021-12-15 13:33:40349 StartNextDecodeForceKeyframe();
Evan Shrubsole476f18d22022-08-15 15:21:16350 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
Evan Shrubsolea0ee64c2022-04-26 08:09:04351 test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()));
Evan Shrubsole9d290262021-12-15 13:33:40352 // Get keyframe. Delta frame should now be scheduled.
Evan Shrubsolea0ee64c2022-04-26 08:09:04353 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
Evan Shrubsole9d290262021-12-15 13:33:40354
355 StartNextDecode();
Evan Shrubsole476f18d22022-08-15 15:21:16356 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder()
357 .Id(1)
358 .Time(kFps30Rtp)
359 .AsLast()
360 .Refs({0})
361 .Build()));
Evan Shrubsole214cab52022-08-16 09:48:23362 buffer_->Stop();
Evan Shrubsole9d290262021-12-15 13:33:40363 // Wait for 2x max wait time. Since we stopped, this should cause no timeouts
364 // or frame-ready callbacks.
Florent Castelli8037fc62024-08-29 13:00:40365 EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame * 2), Eq(std::nullopt));
Evan Shrubsole9d290262021-12-15 13:33:40366}
367
Evan Shrubsole476f18d22022-08-15 15:21:16368TEST_P(VideoStreamBufferControllerTest, FramesWaitForDecoderToComplete) {
Evan Shrubsole9d290262021-12-15 13:33:40369 StartNextDecodeForceKeyframe();
370
371 // Start with a keyframe.
Evan Shrubsole476f18d22022-08-15 15:21:16372 buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
Evan Shrubsolea0ee64c2022-04-26 08:09:04373 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
Evan Shrubsole9d290262021-12-15 13:33:40374
375 ResetLastResult();
376 // Insert a delta frame.
Evan Shrubsole476f18d22022-08-15 15:21:16377 buffer_->InsertFrame(test::FakeFrameBuilder()
378 .Id(1)
379 .Time(kFps30Rtp)
380 .AsLast()
381 .Refs({0})
382 .Build());
Evan Shrubsole9d290262021-12-15 13:33:40383
384 // Advancing time should not result in a frame since the scheduler has not
385 // been signalled that we are ready.
Florent Castelli8037fc62024-08-29 13:00:40386 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Eq(std::nullopt));
Evan Shrubsole9d290262021-12-15 13:33:40387 // Signal ready.
388 StartNextDecode();
Evan Shrubsolea0ee64c2022-04-26 08:09:04389 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1)));
Evan Shrubsole9d290262021-12-15 13:33:40390}
391
Evan Shrubsole476f18d22022-08-15 15:21:16392TEST_P(VideoStreamBufferControllerTest, LateFrameDropped) {
Evan Shrubsole9d290262021-12-15 13:33:40393 StartNextDecodeForceKeyframe();
394 // F1
395 // /
396 // F0 --> F2
Evan Shrubsole476f18d22022-08-15 15:21:16397 buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
Evan Shrubsole9d290262021-12-15 13:33:40398 // Start with a keyframe.
Evan Shrubsolea0ee64c2022-04-26 08:09:04399 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
Evan Shrubsole9d290262021-12-15 13:33:40400
401 StartNextDecode();
402
403 // Simulate late F1 which arrives after F2.
404 time_controller_.AdvanceTime(kFps30Delay * 2);
Evan Shrubsole476f18d22022-08-15 15:21:16405 buffer_->InsertFrame(test::FakeFrameBuilder()
406 .Id(2)
407 .Time(2 * kFps30Rtp)
408 .AsLast()
409 .Refs({0})
410 .Build());
Evan Shrubsolea0ee64c2022-04-26 08:09:04411 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2)));
Evan Shrubsole9d290262021-12-15 13:33:40412
413 StartNextDecode();
414
Evan Shrubsole476f18d22022-08-15 15:21:16415 buffer_->InsertFrame(test::FakeFrameBuilder()
416 .Id(1)
417 .Time(1 * kFps30Rtp)
418 .AsLast()
419 .Refs({0})
420 .Build());
Evan Shrubsole9d290262021-12-15 13:33:40421 // Confirm frame 1 is never scheduled by timing out.
Evan Shrubsole9a999052021-12-12 14:27:00422 EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut());
Evan Shrubsole9d290262021-12-15 13:33:40423}
424
Evan Shrubsole476f18d22022-08-15 15:21:16425TEST_P(VideoStreamBufferControllerTest, FramesFastForwardOnSystemHalt) {
Evan Shrubsole9d290262021-12-15 13:33:40426 StartNextDecodeForceKeyframe();
427 // F1
428 // /
429 // F0 --> F2
Evan Shrubsole476f18d22022-08-15 15:21:16430 buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
Evan Shrubsole9d290262021-12-15 13:33:40431
432 // Start with a keyframe.
Evan Shrubsolea0ee64c2022-04-26 08:09:04433 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
Evan Shrubsole9d290262021-12-15 13:33:40434
435 time_controller_.AdvanceTime(kFps30Delay);
Evan Shrubsole476f18d22022-08-15 15:21:16436 buffer_->InsertFrame(test::FakeFrameBuilder()
437 .Id(1)
438 .Time(kFps30Rtp)
439 .AsLast()
440 .Refs({0})
441 .Build());
Evan Shrubsole9d290262021-12-15 13:33:40442 time_controller_.AdvanceTime(kFps30Delay);
Evan Shrubsole476f18d22022-08-15 15:21:16443 buffer_->InsertFrame(test::FakeFrameBuilder()
444 .Id(2)
445 .Time(2 * kFps30Rtp)
446 .AsLast()
447 .Refs({0})
448 .Build());
Evan Shrubsole9d290262021-12-15 13:33:40449
450 // Halting time should result in F1 being skipped.
451 time_controller_.AdvanceTime(kFps30Delay * 2);
452 StartNextDecode();
Evan Shrubsolea0ee64c2022-04-26 08:09:04453 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2)));
Evan Shrubsole9d290262021-12-15 13:33:40454 EXPECT_EQ(dropped_frames(), 1);
455}
456
Evan Shrubsole476f18d22022-08-15 15:21:16457TEST_P(VideoStreamBufferControllerTest, ForceKeyFrame) {
Evan Shrubsole9d290262021-12-15 13:33:40458 StartNextDecodeForceKeyframe();
459 // Initial keyframe.
Evan Shrubsole476f18d22022-08-15 15:21:16460 buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
Evan Shrubsolea0ee64c2022-04-26 08:09:04461 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
Evan Shrubsole9d290262021-12-15 13:33:40462
463 StartNextDecodeForceKeyframe();
464
465 // F2 is the next keyframe, and should be extracted since a keyframe was
466 // forced.
Evan Shrubsole476f18d22022-08-15 15:21:16467 buffer_->InsertFrame(test::FakeFrameBuilder()
468 .Id(1)
469 .Time(kFps30Rtp)
470 .AsLast()
471 .Refs({0})
472 .Build());
473 buffer_->InsertFrame(
Evan Shrubsolea0ee64c2022-04-26 08:09:04474 test::FakeFrameBuilder().Id(2).Time(kFps30Rtp * 2).AsLast().Build());
Evan Shrubsole9d290262021-12-15 13:33:40475
Evan Shrubsolea0ee64c2022-04-26 08:09:04476 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay * 3), Frame(test::WithId(2)));
Evan Shrubsole9d290262021-12-15 13:33:40477}
478
Evan Shrubsole476f18d22022-08-15 15:21:16479TEST_P(VideoStreamBufferControllerTest, SlowDecoderDropsTemporalLayers) {
Evan Shrubsole9d290262021-12-15 13:33:40480 StartNextDecodeForceKeyframe();
481 // 2 temporal layers, at 15fps per layer to make 30fps total.
482 // Decoder is slower than 30fps, so last_frame() will be skipped.
483 // F1 --> F3 --> F5
484 // / / /
485 // F0 --> F2 --> F4
Evan Shrubsole476f18d22022-08-15 15:21:16486 buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
Evan Shrubsole9d290262021-12-15 13:33:40487 // Keyframe received.
Evan Shrubsole9d290262021-12-15 13:33:40488 // Don't start next decode until slow delay.
Evan Shrubsolea0ee64c2022-04-26 08:09:04489 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
Evan Shrubsole9d290262021-12-15 13:33:40490
491 time_controller_.AdvanceTime(kFps30Delay);
Evan Shrubsole476f18d22022-08-15 15:21:16492 buffer_->InsertFrame(test::FakeFrameBuilder()
493 .Id(1)
494 .Time(1 * kFps30Rtp)
495 .Refs({0})
496 .AsLast()
497 .Build());
Evan Shrubsole9d290262021-12-15 13:33:40498 time_controller_.AdvanceTime(kFps30Delay);
Evan Shrubsole476f18d22022-08-15 15:21:16499 buffer_->InsertFrame(test::FakeFrameBuilder()
500 .Id(2)
501 .Time(2 * kFps30Rtp)
502 .Refs({0})
503 .AsLast()
504 .Build());
Evan Shrubsole9d290262021-12-15 13:33:40505
506 // Simulate decode taking 3x FPS rate.
507 time_controller_.AdvanceTime(kFps30Delay * 1.5);
508 StartNextDecode();
Evan Shrubsole9d290262021-12-15 13:33:40509 // F2 is the best frame since decoding was so slow that F1 is too old.
Evan Shrubsolea0ee64c2022-04-26 08:09:04510 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay * 2), Frame(test::WithId(2)));
Evan Shrubsole9d290262021-12-15 13:33:40511 EXPECT_EQ(dropped_frames(), 1);
512 time_controller_.AdvanceTime(kFps30Delay / 2);
513
Evan Shrubsole476f18d22022-08-15 15:21:16514 buffer_->InsertFrame(test::FakeFrameBuilder()
515 .Id(3)
516 .Time(3 * kFps30Rtp)
517 .Refs({1, 2})
518 .AsLast()
519 .Build());
Evan Shrubsole9d290262021-12-15 13:33:40520 time_controller_.AdvanceTime(kFps30Delay / 2);
Evan Shrubsole476f18d22022-08-15 15:21:16521 buffer_->InsertFrame(test::FakeFrameBuilder()
522 .Id(4)
523 .Time(4 * kFps30Rtp)
524 .Refs({2})
525 .AsLast()
526 .Build());
Evan Shrubsole9d290262021-12-15 13:33:40527 time_controller_.AdvanceTime(kFps30Delay / 2);
528
529 // F4 is the best frame since decoding was so slow that F1 is too old.
530 time_controller_.AdvanceTime(kFps30Delay);
531 StartNextDecode();
Evan Shrubsolea0ee64c2022-04-26 08:09:04532 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(4)));
Evan Shrubsole9d290262021-12-15 13:33:40533
Evan Shrubsole476f18d22022-08-15 15:21:16534 buffer_->InsertFrame(test::FakeFrameBuilder()
535 .Id(5)
536 .Time(5 * kFps30Rtp)
537 .Refs({3, 4})
538 .AsLast()
539 .Build());
Evan Shrubsole9d290262021-12-15 13:33:40540 time_controller_.AdvanceTime(kFps30Delay / 2);
541
542 // F5 is not decodable since F4 was decoded, so a timeout is expected.
543 time_controller_.AdvanceTime(TimeDelta::Millis(10));
544 StartNextDecode();
Evan Shrubsole9a999052021-12-12 14:27:00545 EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut());
Evan Shrubsole9d290262021-12-15 13:33:40546 // TODO(bugs.webrtc.org/13343): This should be 2 dropped frames since frames 1
547 // and 3 were dropped. However, frame_buffer2 does not mark frame 3 as dropped
548 // which is a bug. Uncomment below when that is fixed for frame_buffer2 is
549 // deleted.
550 // EXPECT_EQ(dropped_frames(), 2);
551}
552
Evan Shrubsole476f18d22022-08-15 15:21:16553TEST_P(VideoStreamBufferControllerTest,
554 NewFrameInsertedWhileWaitingToReleaseFrame) {
Evan Shrubsole9d290262021-12-15 13:33:40555 StartNextDecodeForceKeyframe();
556 // Initial keyframe.
Evan Shrubsole476f18d22022-08-15 15:21:16557 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
Evan Shrubsolea0ee64c2022-04-26 08:09:04558 test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()));
559 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
Evan Shrubsole9d290262021-12-15 13:33:40560
Evan Shrubsole6cd6d8e2022-02-11 14:30:26561 time_controller_.AdvanceTime(kFps30Delay / 2);
Evan Shrubsole476f18d22022-08-15 15:21:16562 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder()
563 .Id(1)
564 .Time(kFps30Rtp)
565 .Refs({0})
566 .AsLast()
567 .Build()));
Evan Shrubsole9d290262021-12-15 13:33:40568 StartNextDecode();
Florent Castelli8037fc62024-08-29 13:00:40569 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Eq(std::nullopt));
Evan Shrubsole9d290262021-12-15 13:33:40570
571 // Scheduler is waiting to deliver Frame 1 now. Insert Frame 2. Frame 1 should
572 // be delivered still.
Evan Shrubsole476f18d22022-08-15 15:21:16573 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder()
574 .Id(2)
575 .Time(kFps30Rtp * 2)
576 .Refs({0})
577 .AsLast()
578 .Build()));
Evan Shrubsolea0ee64c2022-04-26 08:09:04579 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1)));
Evan Shrubsole9d290262021-12-15 13:33:40580}
581
Evan Shrubsole476f18d22022-08-15 15:21:16582TEST_P(VideoStreamBufferControllerTest, SameFrameNotScheduledTwice) {
Evan Shrubsole9d290262021-12-15 13:33:40583 // A frame could be scheduled twice if last_frame() arrive out-of-order but
584 // the older frame is old enough to be fast forwarded.
585 //
586 // 1. F2 arrives and is scheduled.
587 // 2. F3 arrives, but scheduling will not change since F2 is next.
588 // 3. F1 arrives late and scheduling is checked since it is before F2. F1
589 // fast-forwarded since it is older.
590 //
591 // F2 is the best frame, but should only be scheduled once, followed by F3.
592 StartNextDecodeForceKeyframe();
593
594 // First keyframe.
Evan Shrubsole476f18d22022-08-15 15:21:16595 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
Evan Shrubsolea0ee64c2022-04-26 08:09:04596 test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()));
597 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Millis(15)),
598 Frame(test::WithId(0)));
Evan Shrubsole9d290262021-12-15 13:33:40599
600 StartNextDecode();
601
Evan Shrubsole9d290262021-12-15 13:33:40602 // F2 arrives and is scheduled.
Evan Shrubsole476f18d22022-08-15 15:21:16603 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
Evan Shrubsole214cab52022-08-16 09:48:23604 test::FakeFrameBuilder().Id(2).Time(2 * kFps30Rtp).AsLast().Build()));
Evan Shrubsole9d290262021-12-15 13:33:40605
606 // F3 arrives before F2 is extracted.
607 time_controller_.AdvanceTime(kFps30Delay);
Evan Shrubsole476f18d22022-08-15 15:21:16608 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
Evan Shrubsole214cab52022-08-16 09:48:23609 test::FakeFrameBuilder().Id(3).Time(3 * kFps30Rtp).AsLast().Build()));
Evan Shrubsole9d290262021-12-15 13:33:40610
611 // F1 arrives and is fast-forwarded since it is too late.
612 // F2 is already scheduled and should not be rescheduled.
613 time_controller_.AdvanceTime(kFps30Delay / 2);
Evan Shrubsole476f18d22022-08-15 15:21:16614 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
Evan Shrubsole214cab52022-08-16 09:48:23615 test::FakeFrameBuilder().Id(1).Time(1 * kFps30Rtp).AsLast().Build()));
Evan Shrubsole9d290262021-12-15 13:33:40616
Evan Shrubsole214cab52022-08-16 09:48:23617 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2)));
Evan Shrubsole9d290262021-12-15 13:33:40618 StartNextDecode();
619
Evan Shrubsole214cab52022-08-16 09:48:23620 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(3)));
Evan Shrubsole6cd6d8e2022-02-11 14:30:26621 StartNextDecode();
622 EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut());
Evan Shrubsole9d290262021-12-15 13:33:40623 EXPECT_EQ(dropped_frames(), 1);
624}
625
Evan Shrubsole476f18d22022-08-15 15:21:16626TEST_P(VideoStreamBufferControllerTest, TestStatsCallback) {
Evan Shrubsole9d290262021-12-15 13:33:40627 EXPECT_CALL(stats_callback_,
628 OnCompleteFrame(true, kFrameSize, VideoContentType::UNSPECIFIED));
Rasmus Brandt621cb292023-05-24 11:29:14629 EXPECT_CALL(stats_callback_, OnDecodableFrame);
Evan Shrubsole9d290262021-12-15 13:33:40630 EXPECT_CALL(stats_callback_, OnFrameBufferTimingsUpdated);
631
632 // Fake timing having received decoded frame.
Evan Shrubsoled6cdf802022-03-02 14:13:55633 timing_.StopDecodeTimer(TimeDelta::Millis(1), clock_->CurrentTime());
Evan Shrubsole9d290262021-12-15 13:33:40634 StartNextDecodeForceKeyframe();
Evan Shrubsole476f18d22022-08-15 15:21:16635 buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
Evan Shrubsolea0ee64c2022-04-26 08:09:04636 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
Evan Shrubsole9d290262021-12-15 13:33:40637
638 // Flush stats posted on the decode queue.
639 time_controller_.AdvanceTime(TimeDelta::Zero());
640}
641
Evan Shrubsole476f18d22022-08-15 15:21:16642TEST_P(VideoStreamBufferControllerTest,
643 FrameCompleteCalledOnceForDuplicateFrame) {
Evan Shrubsoled4ff12f2022-03-01 09:16:11644 EXPECT_CALL(stats_callback_,
645 OnCompleteFrame(true, kFrameSize, VideoContentType::UNSPECIFIED))
646 .Times(1);
647
648 StartNextDecodeForceKeyframe();
Evan Shrubsole476f18d22022-08-15 15:21:16649 buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
650 buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build());
Evan Shrubsoled4ff12f2022-03-01 09:16:11651 // Flush stats posted on the decode queue.
652 time_controller_.AdvanceTime(TimeDelta::Zero());
653}
654
Evan Shrubsole476f18d22022-08-15 15:21:16655TEST_P(VideoStreamBufferControllerTest,
656 FrameCompleteCalledOnceForSingleTemporalUnit) {
Evan Shrubsoled4ff12f2022-03-01 09:16:11657 StartNextDecodeForceKeyframe();
658
659 // `OnCompleteFrame` should not be called for the first two frames since they
660 // do not complete the temporal layer.
661 EXPECT_CALL(stats_callback_, OnCompleteFrame(_, _, _)).Times(0);
Evan Shrubsole476f18d22022-08-15 15:21:16662 buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).Build());
663 buffer_->InsertFrame(
664 test::FakeFrameBuilder().Id(1).Time(0).Refs({0}).Build());
Evan Shrubsoled4ff12f2022-03-01 09:16:11665 time_controller_.AdvanceTime(TimeDelta::Zero());
666 // Flush stats posted on the decode queue.
667 ::testing::Mock::VerifyAndClearExpectations(&stats_callback_);
668
669 // Note that this frame is not marked as a keyframe since the last spatial
670 // layer has dependencies.
671 EXPECT_CALL(stats_callback_,
672 OnCompleteFrame(false, kFrameSize, VideoContentType::UNSPECIFIED))
673 .Times(1);
Evan Shrubsole476f18d22022-08-15 15:21:16674 buffer_->InsertFrame(
Evan Shrubsolea0ee64c2022-04-26 08:09:04675 test::FakeFrameBuilder().Id(2).Time(0).Refs({0, 1}).AsLast().Build());
Evan Shrubsoled4ff12f2022-03-01 09:16:11676 // Flush stats posted on the decode queue.
677 time_controller_.AdvanceTime(TimeDelta::Zero());
678}
679
Evan Shrubsole476f18d22022-08-15 15:21:16680TEST_P(VideoStreamBufferControllerTest,
681 FrameCompleteCalledOnceForCompleteTemporalUnit) {
Evan Shrubsoled4ff12f2022-03-01 09:16:11682 // FrameBuffer2 logs the complete frame on the arrival of the last layer.
Evan Shrubsoled4ff12f2022-03-01 09:16:11683 StartNextDecodeForceKeyframe();
684
685 // `OnCompleteFrame` should not be called for the first two frames since they
686 // do not complete the temporal layer. Frame 1 arrives later, at which time
687 // this frame can finally be considered complete.
688 EXPECT_CALL(stats_callback_, OnCompleteFrame(_, _, _)).Times(0);
Evan Shrubsole476f18d22022-08-15 15:21:16689 buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).Build());
690 buffer_->InsertFrame(
Evan Shrubsolea0ee64c2022-04-26 08:09:04691 test::FakeFrameBuilder().Id(2).Time(0).Refs({0, 1}).AsLast().Build());
Evan Shrubsoled4ff12f2022-03-01 09:16:11692 time_controller_.AdvanceTime(TimeDelta::Zero());
693 // Flush stats posted on the decode queue.
694 ::testing::Mock::VerifyAndClearExpectations(&stats_callback_);
695
696 EXPECT_CALL(stats_callback_,
697 OnCompleteFrame(false, kFrameSize, VideoContentType::UNSPECIFIED))
698 .Times(1);
Evan Shrubsole476f18d22022-08-15 15:21:16699 buffer_->InsertFrame(
700 test::FakeFrameBuilder().Id(1).Time(0).Refs({0}).Build());
Evan Shrubsoled4ff12f2022-03-01 09:16:11701 // Flush stats posted on the decode queue.
702 time_controller_.AdvanceTime(TimeDelta::Zero());
703}
704
Evan Shrubsole6cd6d8e2022-02-11 14:30:26705// Note: This test takes a long time to run if the fake metronome is active.
706// Since the test needs to wait for the timestamp to rollover, it has a fake
707// delay of around 6.5 hours. Even though time is simulated, this will be
708// around 1,500,000 metronome tick invocations.
Evan Shrubsole476f18d22022-08-15 15:21:16709TEST_P(VideoStreamBufferControllerTest, NextFrameWithOldTimestamp) {
Evan Shrubsole9a999052021-12-12 14:27:00710 // Test inserting 31 frames and pause the stream for a long time before
711 // frame 32.
712 StartNextDecodeForceKeyframe();
713 constexpr uint32_t kBaseRtp = std::numeric_limits<uint32_t>::max() / 2;
714
715 // First keyframe. The receive time must be explicitly set in this test since
716 // the RTP derived time used in all tests does not work when the long pause
717 // happens later in the test.
Evan Shrubsole476f18d22022-08-15 15:21:16718 buffer_->InsertFrame(test::FakeFrameBuilder()
719 .Id(0)
720 .Time(kBaseRtp)
721 .ReceivedTime(clock_->CurrentTime())
722 .AsLast()
723 .Build());
Evan Shrubsolea0ee64c2022-04-26 08:09:04724 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(0)));
Evan Shrubsole9a999052021-12-12 14:27:00725
726 // 1 more frame to warmup VCMTiming for 30fps.
727 StartNextDecode();
Evan Shrubsole476f18d22022-08-15 15:21:16728 buffer_->InsertFrame(test::FakeFrameBuilder()
729 .Id(1)
730 .Time(kBaseRtp + kFps30Rtp)
731 .ReceivedTime(clock_->CurrentTime())
732 .AsLast()
733 .Build());
Evan Shrubsolea0ee64c2022-04-26 08:09:04734 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1)));
Evan Shrubsole9a999052021-12-12 14:27:00735
736 // Pause the stream for such a long time it incurs an RTP timestamp rollover
737 // by over half.
738 constexpr uint32_t kLastRtp = kBaseRtp + kFps30Rtp;
739 constexpr uint32_t kRolloverRtp =
740 kLastRtp + std::numeric_limits<uint32_t>::max() / 2 + 1;
741 constexpr Frequency kRtpHz = Frequency::KiloHertz(90);
742 // Pause for corresponding delay such that RTP timestamp would increase this
743 // much at 30fps.
744 constexpr TimeDelta kRolloverDelay =
745 (std::numeric_limits<uint32_t>::max() / 2 + 1) / kRtpHz;
746
747 // Avoid timeout being set while waiting for the frame and before the receiver
748 // is ready.
749 ResetLastResult();
Florent Castelli8037fc62024-08-29 13:00:40750 EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), Eq(std::nullopt));
Evan Shrubsole9a999052021-12-12 14:27:00751 time_controller_.AdvanceTime(kRolloverDelay - kMaxWaitForFrame);
752 StartNextDecode();
Evan Shrubsole476f18d22022-08-15 15:21:16753 buffer_->InsertFrame(test::FakeFrameBuilder()
754 .Id(2)
755 .Time(kRolloverRtp)
756 .ReceivedTime(clock_->CurrentTime())
757 .AsLast()
758 .Build());
Evan Shrubsole9a999052021-12-12 14:27:00759 // FrameBuffer2 drops the frame, while FrameBuffer3 will continue the stream.
Evan Shrubsole4c269322022-08-02 08:57:04760 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2)));
Evan Shrubsole9a999052021-12-12 14:27:00761}
762
Evan Shrubsole8fe55792022-10-27 11:05:44763TEST_P(VideoStreamBufferControllerTest,
764 FrameNotSetForDecodedIfFrameBufferBecomesNonDecodable) {
765 // This can happen if the frame buffer receives non-standard input. This test
766 // will simply clear the frame buffer to replicate this.
767 StartNextDecodeForceKeyframe();
768 // Initial keyframe.
769 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
770 test::FakeFrameBuilder().Id(0).Time(0).SpatialLayer(1).AsLast().Build()));
771 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
772
773 // Insert a frame that will become non-decodable.
774 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder()
775 .Id(11)
776 .Time(kFps30Rtp)
777 .Refs({0})
778 .SpatialLayer(1)
779 .AsLast()
780 .Build()));
781 StartNextDecode();
782 // Second layer inserted after last layer for the same frame out-of-order.
783 // This second frame requires some older frame to be decoded and so now the
784 // super-frame is no longer decodable despite already being scheduled.
785 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder()
786 .Id(10)
787 .Time(kFps30Rtp)
788 .SpatialLayer(0)
789 .Refs({2})
790 .Build()));
791 EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut());
792
793 // Ensure that this frame can be decoded later.
794 StartNextDecode();
795 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder()
796 .Id(2)
797 .Time(kFps30Rtp / 2)
798 .SpatialLayer(0)
799 .Refs({0})
800 .AsLast()
801 .Build()));
802 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2)));
803 StartNextDecode();
804 EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(10)));
805}
806
Evan Shrubsolea006ba12022-09-05 10:09:08807INSTANTIATE_TEST_SUITE_P(VideoStreamBufferController,
808 VideoStreamBufferControllerTest,
809 ::testing::Combine(::testing::Bool(),
810 ::testing::Values("")),
811 [](const auto& info) {
812 return std::get<0>(info.param) ? "SyncDecoding"
813 : "UnsyncedDecoding";
814 });
Evan Shrubsole9d290262021-12-15 13:33:40815
Evan Shrubsole476f18d22022-08-15 15:21:16816class LowLatencyVideoStreamBufferControllerTest
817 : public ::testing::Test,
818 public VideoStreamBufferControllerFixture {};
Evan Shrubsolef7a19372022-02-14 13:05:10819
Evan Shrubsole476f18d22022-08-15 15:21:16820TEST_P(LowLatencyVideoStreamBufferControllerTest,
Evan Shrubsolef7a19372022-02-14 13:05:10821 FramesDecodedInstantlyWithLowLatencyRendering) {
822 // Initial keyframe.
823 StartNextDecodeForceKeyframe();
Evan Shrubsoled6cdf802022-03-02 14:13:55824 timing_.set_min_playout_delay(TimeDelta::Zero());
825 timing_.set_max_playout_delay(TimeDelta::Millis(10));
Evan Shrubsolef7a19372022-02-14 13:05:10826 // Playout delay of 0 implies low-latency rendering.
Danil Chapovalovc146b5f2023-08-16 09:42:52827 auto frame = test::FakeFrameBuilder()
828 .Id(0)
829 .Time(0)
Danil Chapovalov7084e1b2023-08-23 11:16:22830 .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(10)})
Danil Chapovalovc146b5f2023-08-16 09:42:52831 .AsLast()
832 .Build();
Evan Shrubsole476f18d22022-08-15 15:21:16833 buffer_->InsertFrame(std::move(frame));
Evan Shrubsolea0ee64c2022-04-26 08:09:04834 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
Evan Shrubsolef7a19372022-02-14 13:05:10835
836 // Delta frame would normally wait here, but should decode at the pacing rate
837 // in low-latency mode.
838 StartNextDecode();
Danil Chapovalovc146b5f2023-08-16 09:42:52839 frame = test::FakeFrameBuilder()
840 .Id(1)
841 .Time(kFps30Rtp)
Danil Chapovalov7084e1b2023-08-23 11:16:22842 .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(10)})
Danil Chapovalovc146b5f2023-08-16 09:42:52843 .AsLast()
844 .Build();
Evan Shrubsole476f18d22022-08-15 15:21:16845 buffer_->InsertFrame(std::move(frame));
Evan Shrubsolef7a19372022-02-14 13:05:10846 // Pacing is set to 16ms in the field trial so we should not decode yet.
Florent Castelli8037fc62024-08-29 13:00:40847 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Eq(std::nullopt));
Evan Shrubsolef7a19372022-02-14 13:05:10848 time_controller_.AdvanceTime(TimeDelta::Millis(16));
Evan Shrubsolea0ee64c2022-04-26 08:09:04849 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1)));
Evan Shrubsolef7a19372022-02-14 13:05:10850}
851
Evan Shrubsole476f18d22022-08-15 15:21:16852TEST_P(LowLatencyVideoStreamBufferControllerTest, ZeroPlayoutDelayFullQueue) {
Evan Shrubsolef7a19372022-02-14 13:05:10853 // Initial keyframe.
854 StartNextDecodeForceKeyframe();
Evan Shrubsoled6cdf802022-03-02 14:13:55855 timing_.set_min_playout_delay(TimeDelta::Zero());
856 timing_.set_max_playout_delay(TimeDelta::Millis(10));
Danil Chapovalovc146b5f2023-08-16 09:42:52857 auto frame = test::FakeFrameBuilder()
858 .Id(0)
859 .Time(0)
Danil Chapovalov7084e1b2023-08-23 11:16:22860 .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(10)})
Danil Chapovalovc146b5f2023-08-16 09:42:52861 .AsLast()
862 .Build();
Evan Shrubsolef7a19372022-02-14 13:05:10863 // Playout delay of 0 implies low-latency rendering.
Evan Shrubsole476f18d22022-08-15 15:21:16864 buffer_->InsertFrame(std::move(frame));
Evan Shrubsolea0ee64c2022-04-26 08:09:04865 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
Evan Shrubsolef7a19372022-02-14 13:05:10866
867 // Queue up 5 frames (configured max queue size for 0-playout delay pacing).
868 for (int id = 1; id <= 6; ++id) {
Danil Chapovalovc146b5f2023-08-16 09:42:52869 frame = test::FakeFrameBuilder()
870 .Id(id)
871 .Time(kFps30Rtp * id)
Danil Chapovalov7084e1b2023-08-23 11:16:22872 .PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(10)})
Danil Chapovalovc146b5f2023-08-16 09:42:52873 .AsLast()
874 .Build();
Evan Shrubsole476f18d22022-08-15 15:21:16875 buffer_->InsertFrame(std::move(frame));
Evan Shrubsolef7a19372022-02-14 13:05:10876 }
877
878 // The queue is at its max size for zero playout delay pacing, so the pacing
879 // should be ignored and the next frame should be decoded instantly.
880 StartNextDecode();
Evan Shrubsolea0ee64c2022-04-26 08:09:04881 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1)));
Evan Shrubsolef7a19372022-02-14 13:05:10882}
883
Evan Shrubsole476f18d22022-08-15 15:21:16884TEST_P(LowLatencyVideoStreamBufferControllerTest,
885 MinMaxDelayZeroLowLatencyMode) {
Evan Shrubsolef7a19372022-02-14 13:05:10886 // Initial keyframe.
887 StartNextDecodeForceKeyframe();
Evan Shrubsoled6cdf802022-03-02 14:13:55888 timing_.set_min_playout_delay(TimeDelta::Zero());
889 timing_.set_max_playout_delay(TimeDelta::Zero());
Evan Shrubsolef7a19372022-02-14 13:05:10890 // Playout delay of 0 implies low-latency rendering.
Danil Chapovalovc146b5f2023-08-16 09:42:52891 auto frame = test::FakeFrameBuilder()
892 .Id(0)
893 .Time(0)
894 .PlayoutDelay(VideoPlayoutDelay::Minimal())
895 .AsLast()
896 .Build();
Evan Shrubsole476f18d22022-08-15 15:21:16897 buffer_->InsertFrame(std::move(frame));
Evan Shrubsolea0ee64c2022-04-26 08:09:04898 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0)));
Evan Shrubsolef7a19372022-02-14 13:05:10899
900 // Delta frame would normally wait here, but should decode at the pacing rate
901 // in low-latency mode.
902 StartNextDecode();
Danil Chapovalovc146b5f2023-08-16 09:42:52903 frame = test::FakeFrameBuilder()
904 .Id(1)
905 .Time(kFps30Rtp)
906 .PlayoutDelay(VideoPlayoutDelay::Minimal())
907 .AsLast()
908 .Build();
Evan Shrubsole476f18d22022-08-15 15:21:16909 buffer_->InsertFrame(std::move(frame));
Evan Shrubsolef7a19372022-02-14 13:05:10910 // The min/max=0 version of low-latency rendering will result in a large
911 // negative decode wait time, so the frame should be ready right away.
Evan Shrubsolea0ee64c2022-04-26 08:09:04912 EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1)));
Evan Shrubsolef7a19372022-02-14 13:05:10913}
914
915INSTANTIATE_TEST_SUITE_P(
Evan Shrubsole476f18d22022-08-15 15:21:16916 VideoStreamBufferController,
917 LowLatencyVideoStreamBufferControllerTest,
Evan Shrubsolea006ba12022-09-05 10:09:08918 ::testing::Combine(
919 ::testing::Bool(),
920 ::testing::Values(
921 "WebRTC-ZeroPlayoutDelay/min_pacing:16ms,max_decode_queue_size:5/",
922 "WebRTC-ZeroPlayoutDelay/"
923 "min_pacing:16ms,max_decode_queue_size:5/")));
Evan Shrubsolef7a19372022-02-14 13:05:10924
philipel7446b602022-10-06 13:49:17925class IncomingTimestampVideoStreamBufferControllerTest
926 : public ::testing::Test,
927 public VideoStreamBufferControllerFixture {};
928
929TEST_P(IncomingTimestampVideoStreamBufferControllerTest,
930 IncomingTimestampOnMarkerBitOnly) {
931 StartNextDecodeForceKeyframe();
932 EXPECT_CALL(timing_, IncomingTimestampMocked)
933 .Times(field_trials_.IsDisabled("WebRTC-IncomingTimestampOnMarkerBitOnly")
934 ? 3
935 : 1);
936 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
937 test::FakeFrameBuilder().Id(0).SpatialLayer(0).Time(0).Build()));
938 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
939 test::FakeFrameBuilder().Id(1).SpatialLayer(1).Time(0).Build()));
940 buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(
941 test::FakeFrameBuilder().Id(2).SpatialLayer(2).Time(0).AsLast().Build()));
942}
943
944INSTANTIATE_TEST_SUITE_P(
945 VideoStreamBufferController,
946 IncomingTimestampVideoStreamBufferControllerTest,
947 ::testing::Combine(
948 ::testing::Bool(),
949 ::testing::Values(
950 "WebRTC-IncomingTimestampOnMarkerBitOnly/Enabled/",
951 "WebRTC-IncomingTimestampOnMarkerBitOnly/Disabled/")));
952
Evan Shrubsole9d290262021-12-15 13:33:40953} // namespace webrtc