Move MaxWaitingTime and associated state to FrameDecodeTiming. This refactor migrates the logic for calculating the maximum time to wait before decoding a frame from VCMTiming to FrameDecodeTiming. - Move MaxWaitingTime and SetLastDecodeScheduledTimestamp from VCMTiming to FrameDecodeTiming. - Move corresponding unit tests for MaxWaitingTime from timing_unittest.cc to frame_decode_timing_unittest.cc. Bug: b/493549134 Change-Id: I6e0b204ebf790e110f43e26e2e73365671662c35 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/470040 Reviewed-by: Rasmus Brandt <brandtr@webrtc.org> Commit-Queue: Åsa Persson <asapersson@webrtc.org> Cr-Commit-Position: refs/heads/main@{#47659}
diff --git a/modules/video_coding/timing/BUILD.gn b/modules/video_coding/timing/BUILD.gn index a9e40f4..fdaaa04 100644 --- a/modules/video_coding/timing/BUILD.gn +++ b/modules/video_coding/timing/BUILD.gn
@@ -111,7 +111,6 @@ "../../../rtc_base:checks", "../../../rtc_base:logging", "../../../rtc_base:macromagic", - "../../../rtc_base/experiments:field_trial_parser", "../../../rtc_base/synchronization:mutex", "../../../system_wrappers", ]
diff --git a/modules/video_coding/timing/timing.cc b/modules/video_coding/timing/timing.cc index e1d8114..758eb09 100644 --- a/modules/video_coding/timing/timing.cc +++ b/modules/video_coding/timing/timing.cc
@@ -23,7 +23,6 @@ #include "modules/video_coding/timing/decode_time_percentile_filter.h" #include "modules/video_coding/timing/timestamp_extrapolator.h" #include "rtc_base/checks.h" -#include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/logging.h" #include "rtc_base/synchronization/mutex.h" #include "system_wrappers/include/clock.h" @@ -31,8 +30,7 @@ namespace webrtc { namespace { -// Default pacing that is used for the low-latency renderer path. -constexpr TimeDelta kZeroPlayoutDelayDefaultMinPacing = TimeDelta::Millis(8); +// Maximum `max_playout_delay` for a stream to use low-latency rendering. constexpr TimeDelta kLowLatencyStreamMaxPlayoutDelayThreshold = TimeDelta::Millis(500); @@ -83,13 +81,7 @@ ts_extrapolator_( std::make_unique<TimestampExtrapolator>(clock_->CurrentTime(), field_trials)), - decode_time_filter_(std::make_unique<DecodeTimePercentileFilter>()), - zero_playout_delay_min_pacing_("min_pacing", - kZeroPlayoutDelayDefaultMinPacing), - last_decode_scheduled_(Timestamp::Zero()) { - ParseFieldTrial({&zero_playout_delay_min_pacing_}, - field_trials.Lookup("WebRTC-ZeroPlayoutDelay")); -} + decode_time_filter_(std::make_unique<DecodeTimePercentileFilter>()) {} void VCMTiming::Reset() { MutexLock lock(&mutex_); @@ -184,39 +176,6 @@ timings_.max_playout_delay); } -void VCMTiming::SetLastDecodeScheduledTimestamp( - Timestamp last_decode_scheduled) { - MutexLock lock(&mutex_); - last_decode_scheduled_ = last_decode_scheduled; -} - -TimeDelta VCMTiming::MaxWaitingTime(Timestamp render_time, - Timestamp now, - bool too_many_frames_queued) const { - MutexLock lock(&mutex_); - - if (render_time.IsZero() && zero_playout_delay_min_pacing_->us() > 0 && - timings_.min_playout_delay.IsZero() && - timings_.max_playout_delay > TimeDelta::Zero()) { - // `render_time` == 0 indicates that the frame should be decoded and - // rendered as soon as possible. However, the decoder can be choked if too - // many frames are sent at once. Therefore, limit the interframe delay to - // `zero_playout_delay_min_pacing_` unless too many frames are queued in - // which case the frames are sent to the decoder at once. - if (too_many_frames_queued) { - return TimeDelta::Zero(); - } - Timestamp earliest_next_decode_start_time = - last_decode_scheduled_ + zero_playout_delay_min_pacing_; - TimeDelta max_wait_time = now >= earliest_next_decode_start_time - ? TimeDelta::Zero() - : earliest_next_decode_start_time - now; - return max_wait_time; - } - return render_time - now - timings_.estimated_max_decode_time - - timings_.render_delay; -} - TimeDelta VCMTiming::TargetVideoDelay() const { MutexLock lock(&mutex_); return timings_.TargetDelay();
diff --git a/modules/video_coding/timing/timing.h b/modules/video_coding/timing/timing.h index ba8c33c..8d88560 100644 --- a/modules/video_coding/timing/timing.h +++ b/modules/video_coding/timing/timing.h
@@ -23,7 +23,6 @@ #include "api/video/video_timing.h" #include "modules/video_coding/timing/decode_time_percentile_filter.h" #include "modules/video_coding/timing/timestamp_extrapolator.h" -#include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/thread_annotations.h" #include "system_wrappers/include/clock.h" @@ -103,17 +102,6 @@ // should be rendered, assuming that the system time currently is `now`. Timestamp RenderTime(uint32_t rtp_timestamp, Timestamp now) const; - // Returns the maximum time in ms that we can wait for a frame to become - // complete before we must pass it to the decoder. render_time==0 indicates - // that the frames should be processed as quickly as possible, with possibly - // only a small delay added to make sure that the decoder is not overloaded. - // In this case, the parameter too_many_frames_queued is used to signal that - // the decode queue is full and that the frame should be decoded as soon as - // possible. - TimeDelta MaxWaitingTime(Timestamp render_time, - Timestamp now, - bool too_many_frames_queued) const; - // Returns the current target delay which is minimum delay + decode time + // render delay. TimeDelta TargetVideoDelay() const; @@ -126,9 +114,6 @@ VideoFrame::RenderParameters RenderParameters() const; - // Updates the last time a frame was scheduled for decoding. - void SetLastDecodeScheduledTimestamp(Timestamp last_decode_scheduled); - private: mutable Mutex mutex_; Clock* const clock_; @@ -141,15 +126,6 @@ VideoDelayTimings timings_ RTC_GUARDED_BY(mutex_); std::optional<int> max_composition_delay_in_frames_ RTC_GUARDED_BY(mutex_); - // Set by the field trial WebRTC-ZeroPlayoutDelay. The parameter min_pacing - // determines the minimum delay between frames scheduled for decoding that is - // used when min playout delay=0 and max playout delay>=0. - FieldTrialParameter<TimeDelta> zero_playout_delay_min_pacing_ - RTC_GUARDED_BY(mutex_); - // Timestamp at which the last frame was scheduled to be sent to the decoder. - // Used only when the RTP header extension playout delay is set to min=0 ms - // which is indicated by a render time set to 0. - Timestamp last_decode_scheduled_ RTC_GUARDED_BY(mutex_); }; } // namespace webrtc
diff --git a/modules/video_coding/timing/timing_unittest.cc b/modules/video_coding/timing/timing_unittest.cc index 87d163a..0575561 100644 --- a/modules/video_coding/timing/timing_unittest.cc +++ b/modules/video_coding/timing/timing_unittest.cc
@@ -141,191 +141,6 @@ EXPECT_THAT(timing.GetTimings(), HasConsistentVideoDelayTimings()); } -TEST(VCMTimingTest, MaxWaitingTimeIsZeroForZeroRenderTime) { - // This is the default path when the RTP playout delay header extension is set - // to min==0 and max==0. - constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. - constexpr TimeDelta kTimeDelta = 1 / Frequency::Hertz(60); - constexpr Timestamp kZeroRenderTime = Timestamp::Zero(); - SimulatedClock clock(kStartTimeUs); - FieldTrials field_trials = CreateTestFieldTrials(); - VCMTiming timing(&clock, field_trials); - timing.set_playout_delay({TimeDelta::Zero(), TimeDelta::Zero()}); - for (int i = 0; i < 10; ++i) { - clock.AdvanceTime(kTimeDelta); - Timestamp now = clock.CurrentTime(); - EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now, - /*too_many_frames_queued=*/false), - TimeDelta::Zero()); - } - // Another frame submitted at the same time also returns a negative max - // waiting time. - Timestamp now = clock.CurrentTime(); - EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now, - /*too_many_frames_queued=*/false), - TimeDelta::Zero()); - // MaxWaitingTime should be less than zero even if there's a burst of frames. - EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now, - /*too_many_frames_queued=*/false), - TimeDelta::Zero()); - EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now, - /*too_many_frames_queued=*/false), - TimeDelta::Zero()); - EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now, - /*too_many_frames_queued=*/false), - TimeDelta::Zero()); - - EXPECT_THAT(timing.GetTimings(), HasConsistentVideoDelayTimings()); -} - -TEST(VCMTimingTest, MaxWaitingTimeZeroDelayPacingExperiment) { - // The minimum pacing is enabled by a field trial and active if the RTP - // playout delay header extension is set to min==0. - constexpr TimeDelta kMinPacing = TimeDelta::Millis(3); - FieldTrials field_trials = - CreateTestFieldTrials("WebRTC-ZeroPlayoutDelay/min_pacing:3ms/"); - constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. - constexpr TimeDelta kTimeDelta = 1 / Frequency::Hertz(60); - constexpr auto kZeroRenderTime = Timestamp::Zero(); - SimulatedClock clock(kStartTimeUs); - VCMTiming timing(&clock, field_trials); - // MaxWaitingTime() returns zero for evenly spaced video frames. - for (int i = 0; i < 10; ++i) { - clock.AdvanceTime(kTimeDelta); - Timestamp now = clock.CurrentTime(); - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, - /*too_many_frames_queued=*/false), - TimeDelta::Zero()); - timing.SetLastDecodeScheduledTimestamp(now); - } - // Another frame submitted at the same time is paced according to the field - // trial setting. - auto now = clock.CurrentTime(); - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, - /*too_many_frames_queued=*/false), - kMinPacing); - // If there's a burst of frames, the wait time is calculated based on next - // decode time. - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, - /*too_many_frames_queued=*/false), - kMinPacing); - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, - /*too_many_frames_queued=*/false), - kMinPacing); - // Allow a few ms to pass, this should be subtracted from the MaxWaitingTime. - constexpr TimeDelta kTwoMs = TimeDelta::Millis(2); - clock.AdvanceTime(kTwoMs); - now = clock.CurrentTime(); - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, - /*too_many_frames_queued=*/false), - kMinPacing - kTwoMs); - // A frame is decoded at the current time, the wait time should be restored to - // pacing delay. - timing.SetLastDecodeScheduledTimestamp(now); - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, - /*too_many_frames_queued=*/false), - kMinPacing); - - EXPECT_THAT(timing.GetTimings(), HasConsistentVideoDelayTimings()); -} - -TEST(VCMTimingTest, DefaultMaxWaitingTimeUnaffectedByPacingExperiment) { - // The minimum pacing is enabled by a field trial but should not have any - // effect if render_time_ms is greater than 0; - FieldTrials field_trials = - CreateTestFieldTrials("WebRTC-ZeroPlayoutDelay/min_pacing:3ms/"); - constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. - const TimeDelta kTimeDelta = TimeDelta::Millis(1000.0 / 60.0); - SimulatedClock clock(kStartTimeUs); - VCMTiming timing(&clock, field_trials); - clock.AdvanceTime(kTimeDelta); - auto now = clock.CurrentTime(); - Timestamp render_time = now + TimeDelta::Millis(30); - // Estimate the internal processing delay from the first frame. - TimeDelta estimated_processing_delay = - (render_time - now) - - timing.MaxWaitingTime(render_time, now, - /*too_many_frames_queued=*/false); - EXPECT_GT(estimated_processing_delay, TimeDelta::Zero()); - - // Any other frame submitted at the same time should be scheduled according to - // its render time. - for (int i = 0; i < 5; ++i) { - render_time += kTimeDelta; - EXPECT_EQ(timing.MaxWaitingTime(render_time, now, - /*too_many_frames_queued=*/false), - render_time - now - estimated_processing_delay); - } - - EXPECT_THAT(timing.GetTimings(), HasConsistentVideoDelayTimings()); -} - -TEST(VCMTimingTest, MaxWaitingTimeReturnsZeroIfTooManyFramesQueuedIsTrue) { - // The minimum pacing is enabled by a field trial and active if the RTP - // playout delay header extension is set to min==0. - constexpr TimeDelta kMinPacing = TimeDelta::Millis(3); - FieldTrials field_trials = - CreateTestFieldTrials("WebRTC-ZeroPlayoutDelay/min_pacing:3ms/"); - constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. - const TimeDelta kTimeDelta = TimeDelta::Millis(1000.0 / 60.0); - constexpr auto kZeroRenderTime = Timestamp::Zero(); - SimulatedClock clock(kStartTimeUs); - VCMTiming timing(&clock, field_trials); - // MaxWaitingTime() returns zero for evenly spaced video frames. - for (int i = 0; i < 10; ++i) { - clock.AdvanceTime(kTimeDelta); - auto now = clock.CurrentTime(); - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, - /*too_many_frames_queued=*/false), - TimeDelta::Zero()); - timing.SetLastDecodeScheduledTimestamp(now); - } - // Another frame submitted at the same time is paced according to the field - // trial setting. - auto now_ms = clock.CurrentTime(); - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now_ms, - /*too_many_frames_queued=*/false), - kMinPacing); - // MaxWaitingTime returns 0 even if there's a burst of frames if - // too_many_frames_queued is set to true. - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now_ms, - /*too_many_frames_queued=*/true), - TimeDelta::Zero()); - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now_ms, - /*too_many_frames_queued=*/true), - TimeDelta::Zero()); - - EXPECT_THAT(timing.GetTimings(), HasConsistentVideoDelayTimings()); -} - -TEST(VCMTimingTest, MaxWaitingTime) { - FieldTrials field_trials = CreateTestFieldTrials(); - SimulatedClock clock(0); - VCMTiming timing(&clock, field_trials); - timing.set_render_delay(kRenderDelay); - UpdateDecodeTimer(timing, clock, kDecodeTime); - - Timestamp on_time = clock.CurrentTime() + kDecodeTime + kRenderDelay; - - // Early frame. - Timestamp render_time = on_time + TimeDelta::Millis(1); - EXPECT_EQ(timing.MaxWaitingTime(render_time, clock.CurrentTime(), - /*too_many_frames_queued=*/false), - TimeDelta::Millis(1)); - - // Exactly on time. - render_time = on_time; - EXPECT_EQ(timing.MaxWaitingTime(render_time, clock.CurrentTime(), - /*too_many_frames_queued=*/false), - TimeDelta::Zero()); - - // Late frame. - render_time = on_time - TimeDelta::Millis(1); - EXPECT_EQ(timing.MaxWaitingTime(render_time, clock.CurrentTime(), - /*too_many_frames_queued=*/false), - TimeDelta::Millis(-1)); -} - TEST(VCMTimingTest, UpdateCurrentDelayCapsWhenOffByMicroseconds) { FieldTrials field_trials = CreateTestFieldTrials(); SimulatedClock clock(0);
diff --git a/video/BUILD.gn b/video/BUILD.gn index 2c90b4c..f9307fd 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn
@@ -353,11 +353,13 @@ "frame_decode_timing.h", ] deps = [ + "../api:field_trials_view", "../api/units:time_delta", "../api/units:timestamp", "../modules/video_coding/timing:timing_module", "../rtc_base:checks", "../rtc_base:logging", + "../rtc_base/experiments:field_trial_parser", "../system_wrappers", ] } @@ -1454,6 +1456,7 @@ ":video", "../api:field_trials", "../api:field_trials_view", + "../api/units:frequency", "../api/units:time_delta", "../api/units:timestamp", "../modules/video_coding/timing:timing_module",
diff --git a/video/frame_decode_timing.cc b/video/frame_decode_timing.cc index 9a50faf..2851502 100644 --- a/video/frame_decode_timing.cc +++ b/video/frame_decode_timing.cc
@@ -14,19 +14,34 @@ #include <cstdint> #include <optional> +#include "api/field_trials_view.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "modules/video_coding/timing/timing.h" #include "rtc_base/checks.h" +#include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/logging.h" #include "system_wrappers/include/clock.h" namespace webrtc { +namespace { -FrameDecodeTiming::FrameDecodeTiming(Clock* clock, VCMTiming const* timing) - : clock_(clock), timing_(timing) { +// Default pacing that is used for the low-latency renderer path. +constexpr TimeDelta kZeroPlayoutDelayDefaultMinPacing = TimeDelta::Millis(8); + +} // namespace + +FrameDecodeTiming::FrameDecodeTiming(Clock* clock, + VCMTiming const* timing, + const FieldTrialsView& field_trials) + : clock_(clock), + timing_(timing), + zero_playout_delay_min_pacing_("min_pacing", + kZeroPlayoutDelayDefaultMinPacing) { RTC_DCHECK(clock_); RTC_DCHECK(timing_); + ParseFieldTrial({&zero_playout_delay_min_pacing_}, + field_trials.Lookup("WebRTC-ZeroPlayoutDelay")); } std::optional<FrameDecodeTiming::FrameSchedule> @@ -37,8 +52,7 @@ RTC_DCHECK_GE(max_wait_for_frame, TimeDelta::Zero()); const Timestamp now = clock_->CurrentTime(); Timestamp render_time = timing_->RenderTime(next_temporal_unit_rtp, now); - TimeDelta max_wait = - timing_->MaxWaitingTime(render_time, now, too_many_frames_queued); + TimeDelta max_wait = MaxWaitingTime(render_time, now, too_many_frames_queued); // If the delay is not too far in the past, or this is the last decodable // frame then it is the best frame to be decoded. Otherwise, fast-forward @@ -61,4 +75,35 @@ .render_time = render_time}; } +void FrameDecodeTiming::SetLastDecodeScheduledTimestamp( + Timestamp last_decode_scheduled) { + last_decode_scheduled_ = last_decode_scheduled; +} + +TimeDelta FrameDecodeTiming::MaxWaitingTime(Timestamp render_time, + Timestamp now, + bool too_many_frames_queued) const { + const VCMTiming::VideoDelayTimings timings = timing_->GetTimings(); + if (render_time.IsZero() && zero_playout_delay_min_pacing_->us() > 0 && + timings.min_playout_delay.IsZero() && + timings.max_playout_delay > TimeDelta::Zero()) { + // `render_time` == 0 indicates that the frame should be decoded and + // rendered as soon as possible. However, the decoder can be choked if too + // many frames are sent at once. Therefore, limit the interframe delay to + // `zero_playout_delay_min_pacing_` unless too many frames are queued in + // which case the frames are sent to the decoder at once. + if (too_many_frames_queued) { + return TimeDelta::Zero(); + } + Timestamp earliest_next_decode_start_time = + last_decode_scheduled_ + zero_playout_delay_min_pacing_; + TimeDelta max_wait_time = now >= earliest_next_decode_start_time + ? TimeDelta::Zero() + : earliest_next_decode_start_time - now; + return max_wait_time; + } + return render_time - now - timings.estimated_max_decode_time - + timings.render_delay; +} + } // namespace webrtc
diff --git a/video/frame_decode_timing.h b/video/frame_decode_timing.h index f4e7106..ad0598b 100644 --- a/video/frame_decode_timing.h +++ b/video/frame_decode_timing.h
@@ -15,16 +15,20 @@ #include <optional> +#include "api/field_trials_view.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "modules/video_coding/timing/timing.h" +#include "rtc_base/experiments/field_trial_parser.h" #include "system_wrappers/include/clock.h" namespace webrtc { class FrameDecodeTiming { public: - FrameDecodeTiming(Clock* clock, VCMTiming const* timing); + FrameDecodeTiming(Clock* clock, + VCMTiming const* timing, + const FieldTrialsView& field_trials); ~FrameDecodeTiming() = default; FrameDecodeTiming(const FrameDecodeTiming&) = delete; FrameDecodeTiming& operator=(const FrameDecodeTiming&) = delete; @@ -44,9 +48,32 @@ TimeDelta max_wait_for_frame, bool too_many_frames_queued); + // Updates the last time a frame was scheduled for decoding. + void SetLastDecodeScheduledTimestamp(Timestamp last_decode_scheduled); + + // Returns the maximum time that we can wait for a frame to become complete + // before it must be passed to the decoder. render_time==0 indicates that the + // frames should be processed as quickly as possible, with possibly only a + // small delay added to make sure that the decoder is not overloaded. + // The parameter too_many_frames_queued is used to signal that the decode + // queue is full and that the frame should be decoded as soon as possible. + TimeDelta MaxWaitingTime(Timestamp render_time, + Timestamp now, + bool too_many_frames_queued) const; + private: Clock* const clock_; VCMTiming const* const timing_; + + // Set by the field trial WebRTC-ZeroPlayoutDelay. The parameter min_pacing + // determines the minimum delay between frames scheduled for decoding that is + // used when min playout delay=0 and max playout delay>=0. + FieldTrialParameter<TimeDelta> zero_playout_delay_min_pacing_; + + // Timestamp at which the last frame was scheduled to be sent to the decoder. + // Used only when the RTP header extension playout delay is set to min=0 ms + // which is indicated by a render time set to 0. + Timestamp last_decode_scheduled_ = Timestamp::Zero(); }; } // namespace webrtc
diff --git a/video/frame_decode_timing_unittest.cc b/video/frame_decode_timing_unittest.cc index 8696321..1e75f44 100644 --- a/video/frame_decode_timing_unittest.cc +++ b/video/frame_decode_timing_unittest.cc
@@ -14,6 +14,7 @@ #include <optional> #include "api/field_trials.h" +#include "api/units/frequency.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "modules/video_coding/timing/timing.h" @@ -34,6 +35,19 @@ constexpr uint32_t kNextRtp = 90000; constexpr uint32_t kLastRtp = 180000; +constexpr Frequency k25Fps = Frequency::Hertz(25); +constexpr TimeDelta kDecodeTime = TimeDelta::Millis(20); +constexpr TimeDelta kRenderDelay = TimeDelta::Millis(15); + +void UpdateDecodeTimer(VCMTiming& timing, + SimulatedClock& clock, + TimeDelta decode_time) { + for (int i = 0; i < k25Fps.hertz(); ++i) { + clock.AdvanceTime(decode_time); + timing.StopDecodeTimer(decode_time, clock.CurrentTime()); + clock.AdvanceTime(1 / k25Fps - decode_time); + } +} class FrameDecodeTimingTest : public ::testing::Test { public: @@ -41,7 +55,7 @@ : field_trials_(CreateTestFieldTrials()), clock_(Timestamp::Millis(1000)), timing_(&clock_, field_trials_), - frame_decode_scheduler_(&clock_, &timing_) { + frame_decode_scheduler_(&clock_, &timing_, field_trials_) { timing_.set_render_delay(TimeDelta::Zero()); timing_.OnCompleteTemporalUnit(kNextRtp, clock_.CurrentTime()); } @@ -119,5 +133,192 @@ Eq(clock_.CurrentTime() + kDecodeDelay))))); } +TEST(FrameDecodeTimingMaxWaitingTimeTest, IsZeroForZeroRenderTime) { + // This is the default path when the RTP playout delay header extension is set + // to min==0 and max==0. + constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. + constexpr TimeDelta kTimeDelta = 1 / Frequency::Hertz(60); + constexpr Timestamp kZeroRenderTime = Timestamp::Zero(); + SimulatedClock clock(kStartTimeUs); + FieldTrials field_trials = CreateTestFieldTrials(); + VCMTiming timing(&clock, field_trials); + timing.set_playout_delay({TimeDelta::Zero(), TimeDelta::Zero()}); + FrameDecodeTiming decode_timing(&clock, &timing, field_trials); + + for (int i = 0; i < 10; ++i) { + clock.AdvanceTime(kTimeDelta); + Timestamp now = clock.CurrentTime(); + EXPECT_LT(decode_timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + } + // Another frame submitted at the same time also returns a negative max + // waiting time. + Timestamp now = clock.CurrentTime(); + EXPECT_LT(decode_timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + // MaxWaitingTime should be less than zero even if there's a burst of frames. + EXPECT_LT(decode_timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + EXPECT_LT(decode_timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + EXPECT_LT(decode_timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); +} + +TEST(FrameDecodeTimingMaxWaitingTimeTest, WithZeroDelayPacingActive) { + // The minimum pacing is enabled by a field trial and active if the RTP + // playout delay header extension is set to min==0. + constexpr TimeDelta kMinPacing = TimeDelta::Millis(3); + FieldTrials field_trials = + CreateTestFieldTrials("WebRTC-ZeroPlayoutDelay/min_pacing:3ms/"); + constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. + constexpr TimeDelta kTimeDelta = 1 / Frequency::Hertz(60); + constexpr Timestamp kZeroRenderTime = Timestamp::Zero(); + SimulatedClock clock(kStartTimeUs); + VCMTiming timing(&clock, field_trials); + FrameDecodeTiming decode_timing(&clock, &timing, field_trials); + + // MaxWaitingTime() returns zero for evenly spaced video frames. + for (int i = 0; i < 10; ++i) { + clock.AdvanceTime(kTimeDelta); + Timestamp now = clock.CurrentTime(); + EXPECT_EQ(decode_timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + decode_timing.SetLastDecodeScheduledTimestamp(now); + } + // Another frame submitted at the same time is paced according to the field + // trial setting. + Timestamp now = clock.CurrentTime(); + EXPECT_EQ(decode_timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + kMinPacing); + // If there's a burst of frames, the wait time is calculated based on next + // decode time. + EXPECT_EQ(decode_timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + kMinPacing); + EXPECT_EQ(decode_timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + kMinPacing); + // Allow a few ms to pass, this should be subtracted from the MaxWaitingTime. + constexpr TimeDelta kTwoMs = TimeDelta::Millis(2); + clock.AdvanceTime(kTwoMs); + now = clock.CurrentTime(); + EXPECT_EQ(decode_timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + kMinPacing - kTwoMs); + // A frame is decoded at the current time, the wait time should be restored to + // pacing delay. + decode_timing.SetLastDecodeScheduledTimestamp(now); + EXPECT_EQ(decode_timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + kMinPacing); +} + +TEST(FrameDecodeTimingMaxWaitingTimeTest, + DefaultMaxWaitingTimeUnaffectedByPacingExperiment) { + // The minimum pacing is enabled by a field trial but should not have any + // effect if render_time is greater than 0; + FieldTrials field_trials = + CreateTestFieldTrials("WebRTC-ZeroPlayoutDelay/min_pacing:3ms/"); + constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. + const TimeDelta kTimeDelta = TimeDelta::Millis(1000.0 / 60.0); + SimulatedClock clock(kStartTimeUs); + VCMTiming timing(&clock, field_trials); + FrameDecodeTiming decode_timing(&clock, &timing, field_trials); + + clock.AdvanceTime(kTimeDelta); + Timestamp now = clock.CurrentTime(); + Timestamp render_time = now + TimeDelta::Millis(30); + // Estimate the internal processing delay from the first frame. + TimeDelta estimated_processing_delay = + (render_time - now) - + decode_timing.MaxWaitingTime(render_time, now, + /*too_many_frames_queued=*/false); + EXPECT_GT(estimated_processing_delay, TimeDelta::Zero()); + + // Any other frame submitted at the same time should be scheduled according to + // its render time. + for (int i = 0; i < 5; ++i) { + render_time += kTimeDelta; + EXPECT_EQ(decode_timing.MaxWaitingTime(render_time, now, + /*too_many_frames_queued=*/false), + render_time - now - estimated_processing_delay); + } +} + +TEST(FrameDecodeTimingMaxWaitingTimeTest, ReturnsZeroIfTooManyFramesAreQueued) { + // The minimum pacing is enabled by a field trial and active if the RTP + // playout delay header extension is set to min==0. + constexpr TimeDelta kMinPacing = TimeDelta::Millis(3); + FieldTrials field_trials = + CreateTestFieldTrials("WebRTC-ZeroPlayoutDelay/min_pacing:3ms/"); + constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. + const TimeDelta kTimeDelta = TimeDelta::Millis(1000.0 / 60.0); + constexpr Timestamp kZeroRenderTime = Timestamp::Zero(); + SimulatedClock clock(kStartTimeUs); + VCMTiming timing(&clock, field_trials); + FrameDecodeTiming decode_timing(&clock, &timing, field_trials); + + // MaxWaitingTime() returns zero for evenly spaced video frames. + for (int i = 0; i < 10; ++i) { + clock.AdvanceTime(kTimeDelta); + Timestamp now = clock.CurrentTime(); + EXPECT_EQ(decode_timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + decode_timing.SetLastDecodeScheduledTimestamp(now); + } + // Another frame submitted at the same time is paced according to the field + // trial setting. + Timestamp now_ms = clock.CurrentTime(); + EXPECT_EQ(decode_timing.MaxWaitingTime(kZeroRenderTime, now_ms, + /*too_many_frames_queued=*/false), + kMinPacing); + // MaxWaitingTime returns 0 even if there's a burst of frames if + // too_many_frames_queued is set to true. + EXPECT_EQ(decode_timing.MaxWaitingTime(kZeroRenderTime, now_ms, + /*too_many_frames_queued=*/true), + TimeDelta::Zero()); + EXPECT_EQ(decode_timing.MaxWaitingTime(kZeroRenderTime, now_ms, + /*too_many_frames_queued=*/true), + TimeDelta::Zero()); +} + +TEST(FrameDecodeTimingMaxWaitingTimeTest, WithVaryingRenderTimes) { + FieldTrials field_trials = CreateTestFieldTrials(); + SimulatedClock clock(0); + VCMTiming timing(&clock, field_trials); + timing.set_render_delay(kRenderDelay); + UpdateDecodeTimer(timing, clock, kDecodeTime); + FrameDecodeTiming decode_timing(&clock, &timing, field_trials); + + Timestamp on_time = clock.CurrentTime() + kDecodeTime + kRenderDelay; + + // Early frame. + Timestamp render_time = on_time + TimeDelta::Millis(1); + EXPECT_EQ(decode_timing.MaxWaitingTime(render_time, clock.CurrentTime(), + /*too_many_frames_queued=*/false), + TimeDelta::Millis(1)); + + // Exactly on time. + render_time = on_time; + EXPECT_EQ(decode_timing.MaxWaitingTime(render_time, clock.CurrentTime(), + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + + // Late frame. + render_time = on_time - TimeDelta::Millis(1); + EXPECT_EQ(decode_timing.MaxWaitingTime(render_time, clock.CurrentTime(), + /*too_many_frames_queued=*/false), + TimeDelta::Millis(-1)); +} + } // namespace } // namespace webrtc
diff --git a/video/video_stream_buffer_controller.cc b/video/video_stream_buffer_controller.cc index edf78d1..713fac6 100644 --- a/video/video_stream_buffer_controller.cc +++ b/video/video_stream_buffer_controller.cc
@@ -110,7 +110,7 @@ buffer_(std::make_unique<FrameBuffer>(kMaxFramesBuffered, kMaxFramesHistory, field_trials)), - decode_timing_(clock_, timing_), + decode_timing_(clock_, timing_, field_trials_), timeout_tracker_( clock_, worker_queue, @@ -265,7 +265,7 @@ std::unique_ptr<EncodedFrame> frame = CombineAndDeleteFrames(std::move(frames)); - timing_->SetLastDecodeScheduledTimestamp(now); + decode_timing_.SetLastDecodeScheduledTimestamp(now); decoder_ready_for_new_frame_ = false; receiver_->OnEncodedFrame(std::move(frame));