Group members in VCMTiming into a VideoDelayTimings struct. Bug: b/493549134 Change-Id: I8af7d30ecb49bc0180c6d631166277052cfb1d8a Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/461220 Commit-Queue: Åsa Persson <asapersson@webrtc.org> Reviewed-by: Rasmus Brandt <brandtr@webrtc.org> Cr-Commit-Position: refs/heads/main@{#47306}
diff --git a/modules/video_coding/timing/timing.cc b/modules/video_coding/timing/timing.cc index 79f71f3..7bea914 100644 --- a/modules/video_coding/timing/timing.cc +++ b/modules/video_coding/timing/timing.cc
@@ -31,7 +31,6 @@ namespace webrtc { namespace { -constexpr TimeDelta kDefaultRenderDelay = TimeDelta::Millis(10); // Default pacing that is used for the low-latency renderer path. constexpr TimeDelta kZeroPlayoutDelayDefaultMinPacing = TimeDelta::Millis(8); constexpr TimeDelta kLowLatencyStreamMaxPlayoutDelayThreshold = @@ -54,18 +53,26 @@ } // namespace +void VCMTiming::VideoDelayTimings::Reset() { + minimum_delay = TimeDelta::Zero(); + estimated_max_decode_time = TimeDelta::Zero(); + render_delay = kDefaultRenderDelay; + min_playout_delay = TimeDelta::Zero(); + target_delay = TimeDelta::Zero(); + current_delay = TimeDelta::Zero(); +} + +bool VCMTiming::VideoDelayTimings::UseLowLatencyRendering() const { + return min_playout_delay.IsZero() && + max_playout_delay <= kLowLatencyStreamMaxPlayoutDelayThreshold; +} + VCMTiming::VCMTiming(Clock* clock, const FieldTrialsView& field_trials) : clock_(clock), ts_extrapolator_( std::make_unique<TimestampExtrapolator>(clock_->CurrentTime(), field_trials)), decode_time_filter_(std::make_unique<DecodeTimePercentileFilter>()), - render_delay_(kDefaultRenderDelay), - min_playout_delay_(TimeDelta::Zero()), - max_playout_delay_(TimeDelta::Seconds(10)), - jitter_delay_(TimeDelta::Zero()), - current_delay_(TimeDelta::Zero()), - num_decoded_frames_(0), zero_playout_delay_min_pacing_("min_pacing", kZeroPlayoutDelayDefaultMinPacing), last_decode_scheduled_(Timestamp::Zero()) { @@ -77,27 +84,24 @@ MutexLock lock(&mutex_); ts_extrapolator_->Reset(clock_->CurrentTime()); decode_time_filter_ = std::make_unique<DecodeTimePercentileFilter>(); - render_delay_ = kDefaultRenderDelay; - min_playout_delay_ = TimeDelta::Zero(); - jitter_delay_ = TimeDelta::Zero(); - current_delay_ = TimeDelta::Zero(); + timings_.Reset(); } void VCMTiming::set_render_delay(TimeDelta render_delay) { MutexLock lock(&mutex_); - render_delay_ = render_delay; + timings_.render_delay = render_delay; } TimeDelta VCMTiming::min_playout_delay() const { MutexLock lock(&mutex_); - return min_playout_delay_; + return timings_.min_playout_delay; } void VCMTiming::set_min_playout_delay(TimeDelta min_playout_delay) { MutexLock lock(&mutex_); - if (min_playout_delay_ != min_playout_delay) { - CheckDelaysValid(min_playout_delay, max_playout_delay_); - min_playout_delay_ = min_playout_delay; + if (timings_.min_playout_delay != min_playout_delay) { + CheckDelaysValid(min_playout_delay, timings_.max_playout_delay); + timings_.min_playout_delay = min_playout_delay; } } @@ -105,17 +109,17 @@ MutexLock lock(&mutex_); // No need to call `CheckDelaysValid` as the same invariant (min <= max) // is guaranteed by the `VideoPlayoutDelay` type. - min_playout_delay_ = playout_delay.min(); - max_playout_delay_ = playout_delay.max(); + timings_.min_playout_delay = playout_delay.min(); + timings_.max_playout_delay = playout_delay.max(); } -void VCMTiming::SetJitterDelay(TimeDelta jitter_delay) { +void VCMTiming::SetJitterDelay(TimeDelta minimum_delay) { MutexLock lock(&mutex_); - if (jitter_delay != jitter_delay_) { - jitter_delay_ = jitter_delay; + if (minimum_delay != timings_.minimum_delay) { + timings_.minimum_delay = minimum_delay; // When in initial state, set current delay to minimum delay. - if (current_delay_.IsZero()) { - current_delay_ = jitter_delay_; + if (timings_.current_delay.IsZero()) { + timings_.current_delay = timings_.minimum_delay; } } } @@ -125,16 +129,16 @@ MutexLock lock(&mutex_); TimeDelta target_delay = TargetDelayInternal(); TimeDelta delayed = (actual_decode_time - render_time) + - EstimatedMaxDecodeTime() + render_delay_; + EstimatedMaxDecodeTime() + timings_.render_delay; // Only consider `delayed` as negative by more than a few microseconds. if (delayed.ms() < 0) { return; } - if (current_delay_ + delayed <= target_delay) { - current_delay_ += delayed; + if (timings_.current_delay + delayed <= target_delay) { + timings_.current_delay += delayed; } else { - current_delay_ = target_delay; + timings_.current_delay = target_delay; } } @@ -142,7 +146,7 @@ MutexLock lock(&mutex_); decode_time_filter_->AddTiming(decode_time.ms(), now.ms()); RTC_DCHECK_GE(decode_time, TimeDelta::Zero()); - ++num_decoded_frames_; + ++timings_.num_decoded_frames; } void VCMTiming::IncomingTimestamp(uint32_t rtp_timestamp, Timestamp now) { @@ -163,7 +167,7 @@ Timestamp VCMTiming::RenderTimeInternal(uint32_t frame_timestamp, Timestamp now) const { - if (UseLowLatencyRendering()) { + if (timings_.UseLowLatencyRendering()) { // Render as soon as possible or with low-latency renderer algorithm. return Timestamp::Zero(); } @@ -176,10 +180,11 @@ } Timestamp estimated_complete_time = *local_time; - // Make sure the actual delay stays in the range of `min_playout_delay_` - // and `max_playout_delay_`. + // Make sure the actual delay stays in the range of `min_playout_delay` + // and `max_playout_delay`. TimeDelta actual_delay = - std::clamp(current_delay_, min_playout_delay_, max_playout_delay_); + std::clamp(timings_.current_delay, timings_.min_playout_delay, + timings_.max_playout_delay); return estimated_complete_time + actual_delay; } @@ -195,7 +200,8 @@ MutexLock lock(&mutex_); if (render_time.IsZero() && zero_playout_delay_min_pacing_->us() > 0 && - min_playout_delay_.IsZero() && max_playout_delay_ > TimeDelta::Zero()) { + timings_.min_playout_delay.IsZero() && + timings_.max_playout_delay > TimeDelta::Zero()) { // `render_time` == 0 indicates that the frame should be decoded and // rendered as soon as possible. However, the decoder can be choked if too // many frames are sent at once. Therefore, limit the interframe delay to @@ -211,7 +217,7 @@ : earliest_next_decode_start_time - now; return max_wait_time; } - return render_time - now - EstimatedMaxDecodeTime() - render_delay_; + return render_time - now - EstimatedMaxDecodeTime() - timings_.render_delay; } TimeDelta VCMTiming::TargetVideoDelay() const { @@ -220,43 +226,32 @@ } TimeDelta VCMTiming::TargetDelayInternal() const { - return std::max(min_playout_delay_, - jitter_delay_ + EstimatedMaxDecodeTime() + render_delay_); + return std::max(timings_.min_playout_delay, timings_.minimum_delay + + EstimatedMaxDecodeTime() + + timings_.render_delay); } // TODO(crbug.com/webrtc/15197): Centralize delay arithmetic. TimeDelta VCMTiming::StatsTargetDelayInternal() const { TimeDelta stats_target_delay = - TargetDelayInternal() - (EstimatedMaxDecodeTime() + render_delay_); + TargetDelayInternal() - + (EstimatedMaxDecodeTime() + timings_.render_delay); return std::max(TimeDelta::Zero(), stats_target_delay); } VideoFrame::RenderParameters VCMTiming::RenderParameters() const { MutexLock lock(&mutex_); - return {.use_low_latency_rendering = UseLowLatencyRendering(), + return {.use_low_latency_rendering = timings_.UseLowLatencyRendering(), .max_composition_delay_in_frames = max_composition_delay_in_frames_}; } -bool VCMTiming::UseLowLatencyRendering() const { - // min_playout_delay_==0, - // max_playout_delay_<=kLowLatencyStreamMaxPlayoutDelayThreshold indicates - // that the low-latency path should be used, which means that frames should be - // decoded and rendered as soon as possible. - return min_playout_delay_.IsZero() && - max_playout_delay_ <= kLowLatencyStreamMaxPlayoutDelayThreshold; -} - VCMTiming::VideoDelayTimings VCMTiming::GetTimings() const { MutexLock lock(&mutex_); - return VideoDelayTimings{ - .num_decoded_frames = num_decoded_frames_, - .minimum_delay = jitter_delay_, - .estimated_max_decode_time = EstimatedMaxDecodeTime(), - .render_delay = render_delay_, - .min_playout_delay = min_playout_delay_, - .max_playout_delay = max_playout_delay_, - .target_delay = StatsTargetDelayInternal(), - .current_delay = current_delay_}; + VideoDelayTimings timings = timings_; + // TODO(b/493549134): Update in StopDecodeTimer. + timings.estimated_max_decode_time = EstimatedMaxDecodeTime(); + timings.target_delay = StatsTargetDelayInternal(); + return timings; } void VCMTiming::SetMaxCompositionDelayInFrames(
diff --git a/modules/video_coding/timing/timing.h b/modules/video_coding/timing/timing.h index 9ee0d8a..5ae772a 100644 --- a/modules/video_coding/timing/timing.h +++ b/modules/video_coding/timing/timing.h
@@ -33,27 +33,34 @@ class VCMTiming { public: struct VideoDelayTimings { - size_t num_decoded_frames; + static constexpr TimeDelta kDefaultRenderDelay = TimeDelta::Millis(10); + + void Reset(); + // Returns whether the low-latency path should be used, i.e., frames should + // be decoded and rendered as soon as possible. + bool UseLowLatencyRendering() const; + + size_t num_decoded_frames = 0; // Pre-decode delay added to smooth out frame delay variation ("jitter") // caused by the network. The target delay will be no smaller than this // delay, thus it is called `minimum_delay`. - TimeDelta minimum_delay; + TimeDelta minimum_delay = TimeDelta::Zero(); // Estimated time needed to decode a video frame. Obtained as the 95th // percentile decode time over a recent time window. - TimeDelta estimated_max_decode_time; + TimeDelta estimated_max_decode_time = TimeDelta::Zero(); // Post-decode delay added to smooth out frame delay variation caused by // decoding and rendering. Set to a constant. - TimeDelta render_delay; + TimeDelta render_delay = kDefaultRenderDelay; // Minimum total delay used when determining render time for a frame. // Obtained from API, `playout-delay` RTP header extension, or A/V sync. - TimeDelta min_playout_delay; + TimeDelta min_playout_delay = TimeDelta::Zero(); // Maximum total delay used when determining render time for a frame. // Obtained from `playout-delay` RTP header extension. - TimeDelta max_playout_delay; + TimeDelta max_playout_delay = TimeDelta::Seconds(10); // Target total delay. Obtained from all the elements above. - TimeDelta target_delay; + TimeDelta target_delay = TimeDelta::Zero(); // Current total delay. Obtained by smoothening the `target_delay`. - TimeDelta current_delay; + TimeDelta current_delay = TimeDelta::Zero(); }; VCMTiming(Clock* clock, const FieldTrialsView& field_trials); @@ -129,7 +136,6 @@ TimeDelta TargetDelayInternal() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); TimeDelta StatsTargetDelayInternal() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); - bool UseLowLatencyRendering() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); mutable Mutex mutex_; Clock* const clock_; @@ -137,17 +143,10 @@ RTC_PT_GUARDED_BY(mutex_); std::unique_ptr<DecodeTimePercentileFilter> decode_time_filter_ RTC_GUARDED_BY(mutex_) RTC_PT_GUARDED_BY(mutex_); - TimeDelta render_delay_ RTC_GUARDED_BY(mutex_); - // Best-effort playout delay range for frames from capture to render. - // The receiver tries to keep the delay between `min_playout_delay_ms_` - // and `max_playout_delay_ms_` taking the network jitter into account. - // A special case is where min_playout_delay_ms_ = max_playout_delay_ms_ = 0, - // in which case the receiver tries to play the frames as they arrive. - TimeDelta min_playout_delay_ RTC_GUARDED_BY(mutex_); - TimeDelta max_playout_delay_ RTC_GUARDED_BY(mutex_); - TimeDelta jitter_delay_ RTC_GUARDED_BY(mutex_); - TimeDelta current_delay_ RTC_GUARDED_BY(mutex_); - size_t num_decoded_frames_ RTC_GUARDED_BY(mutex_); + + // Holds the current video delay timings. + VideoDelayTimings timings_ RTC_GUARDED_BY(mutex_); + std::optional<int> max_composition_delay_in_frames_ RTC_GUARDED_BY(mutex_); // Set by the field trial WebRTC-ZeroPlayoutDelay. The parameter min_pacing // determines the minimum delay between frames scheduled for decoding that is
diff --git a/modules/video_coding/timing/timing_unittest.cc b/modules/video_coding/timing/timing_unittest.cc index afe11d0..b2ea1ad 100644 --- a/modules/video_coding/timing/timing_unittest.cc +++ b/modules/video_coding/timing/timing_unittest.cc
@@ -119,7 +119,6 @@ FieldTrials field_trials = CreateTestFieldTrials(); SimulatedClock clock(0); VCMTiming timing(&clock, field_trials); - timing.Reset(); // Default is false. EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering); // False if min playout delay > 0. @@ -150,7 +149,6 @@ SimulatedClock clock(kStartTimeUs); FieldTrials field_trials = CreateTestFieldTrials(); VCMTiming timing(&clock, field_trials); - timing.Reset(); timing.set_playout_delay({TimeDelta::Zero(), TimeDelta::Zero()}); for (int i = 0; i < 10; ++i) { clock.AdvanceTime(kTimeDelta); @@ -190,7 +188,6 @@ constexpr auto kZeroRenderTime = Timestamp::Zero(); SimulatedClock clock(kStartTimeUs); VCMTiming timing(&clock, field_trials); - timing.Reset(); // MaxWaitingTime() returns zero for evenly spaced video frames. for (int i = 0; i < 10; ++i) { clock.AdvanceTime(kTimeDelta); @@ -240,7 +237,6 @@ const TimeDelta kTimeDelta = TimeDelta::Millis(1000.0 / 60.0); SimulatedClock clock(kStartTimeUs); VCMTiming timing(&clock, field_trials); - timing.Reset(); clock.AdvanceTime(kTimeDelta); auto now = clock.CurrentTime(); Timestamp render_time = now + TimeDelta::Millis(30); @@ -274,7 +270,6 @@ constexpr auto kZeroRenderTime = Timestamp::Zero(); SimulatedClock clock(kStartTimeUs); VCMTiming timing(&clock, field_trials); - timing.Reset(); // MaxWaitingTime() returns zero for evenly spaced video frames. for (int i = 0; i < 10; ++i) { clock.AdvanceTime(kTimeDelta); @@ -334,7 +329,6 @@ FieldTrials field_trials = CreateTestFieldTrials(); SimulatedClock clock(0); VCMTiming timing(&clock, field_trials); - timing.Reset(); // Set larger initial current delay. timing.set_min_playout_delay(TimeDelta::Millis(200)); @@ -353,6 +347,23 @@ // EXPECT_THAT(timing.GetTimings(), HasConsistentVideoDelayTimings()); } +TEST(VCMTimingTest, InitialVideoDelayTimings) { + FieldTrials field_trials = CreateTestFieldTrials(); + SimulatedClock clock(0); + VCMTiming timing(&clock, field_trials); + + VCMTiming::VideoDelayTimings timings = timing.GetTimings(); + EXPECT_EQ(timings.num_decoded_frames, 0u); + EXPECT_EQ(timings.minimum_delay, TimeDelta::Zero()); + EXPECT_EQ(timings.estimated_max_decode_time, TimeDelta::Zero()); + EXPECT_EQ(timings.render_delay, + VCMTiming::VideoDelayTimings::kDefaultRenderDelay); + EXPECT_EQ(timings.min_playout_delay, TimeDelta::Zero()); + EXPECT_EQ(timings.target_delay, TimeDelta::Zero()); + EXPECT_EQ(timings.current_delay, TimeDelta::Zero()); + EXPECT_THAT(timings, HasConsistentVideoDelayTimings()); +} + TEST(VCMTimingTest, GetTimings) { FieldTrials field_trials = CreateTestFieldTrials(); SimulatedClock clock(33); @@ -397,8 +408,7 @@ SimulatedClock clock(Timestamp::Millis(33)); VCMTiming timing(&clock, field_trials); - const TimeDelta default_render_delay = timing.GetTimings().render_delay; - timing.set_render_delay(default_render_delay + TimeDelta::Millis(1)); + timing.set_render_delay(TimeDelta::Millis(11)); TimeDelta min_playout_delay = TimeDelta::Millis(50); TimeDelta max_playout_delay = TimeDelta::Millis(500); timing.set_playout_delay({min_playout_delay, max_playout_delay}); @@ -420,7 +430,8 @@ EXPECT_GT(timings.num_decoded_frames, 0u); EXPECT_EQ(timings.minimum_delay, TimeDelta::Zero()); EXPECT_EQ(timings.estimated_max_decode_time, TimeDelta::Zero()); - EXPECT_EQ(timings.render_delay, default_render_delay); + EXPECT_EQ(timings.render_delay, + VCMTiming::VideoDelayTimings::kDefaultRenderDelay); EXPECT_EQ(timings.min_playout_delay, TimeDelta::Zero()); EXPECT_EQ(timings.max_playout_delay, max_playout_delay); EXPECT_EQ(timings.target_delay, TimeDelta::Zero());