diff --git a/modules/video_coding/frame_buffer2.cc b/modules/video_coding/frame_buffer2.cc
index 903b9fb..ba77b72 100644
--- a/modules/video_coding/frame_buffer2.cc
+++ b/modules/video_coding/frame_buffer2.cc
@@ -110,6 +110,8 @@
           if (!frames_to_decode_.empty()) {
             // We have frames, deliver!
             frame = absl::WrapUnique(GetNextFrame());
+            timing_->SetLastDecodeScheduledTimestamp(
+                clock_->TimeInMilliseconds());
           } else if (clock_->TimeInMilliseconds() < latest_return_time_ms_) {
             // If there's no frames to decode and there is still time left, it
             // means that the frame buffer was cleared between creation and
diff --git a/modules/video_coding/frame_buffer2_unittest.cc b/modules/video_coding/frame_buffer2_unittest.cc
index d37efda..68acf81 100644
--- a/modules/video_coding/frame_buffer2_unittest.cc
+++ b/modules/video_coding/frame_buffer2_unittest.cc
@@ -55,7 +55,8 @@
     return last_ms_;
   }
 
-  int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) override {
+  int64_t MaxWaitingTime(int64_t render_time_ms,
+                         int64_t now_ms) const override {
     return render_time_ms - now_ms - kDecodeTime;
   }
 
diff --git a/modules/video_coding/timing.cc b/modules/video_coding/timing.cc
index 748013c..dc25efe 100644
--- a/modules/video_coding/timing.cc
+++ b/modules/video_coding/timing.cc
@@ -35,7 +35,7 @@
       num_decoded_frames_(0),
       low_latency_renderer_enabled_("enabled", true),
       zero_playout_delay_min_pacing_("min_pacing", TimeDelta::Millis(0)),
-      earliest_next_decode_start_time_(0) {
+      last_decode_scheduled_ts_(0) {
   ParseFieldTrial({&low_latency_renderer_enabled_},
                   field_trial::FindFullName("WebRTC-LowLatencyRenderer"));
   ParseFieldTrial({&zero_playout_delay_min_pacing_},
@@ -171,6 +171,12 @@
   return RenderTimeMsInternal(frame_timestamp, now_ms);
 }
 
+void VCMTiming::SetLastDecodeScheduledTimestamp(
+    int64_t last_decode_scheduled_ts) {
+  MutexLock lock(&mutex_);
+  last_decode_scheduled_ts_ = last_decode_scheduled_ts;
+}
+
 int64_t VCMTiming::RenderTimeMsInternal(uint32_t frame_timestamp,
                                         int64_t now_ms) const {
   constexpr int kLowLatencyRendererMaxPlayoutDelayMs = 500;
@@ -202,7 +208,8 @@
   return decode_time_ms;
 }
 
-int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) {
+int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms,
+                                  int64_t now_ms) const {
   MutexLock lock(&mutex_);
 
   if (render_time_ms == 0 && zero_playout_delay_min_pacing_->us() > 0) {
@@ -210,11 +217,11 @@
     // rendered as soon as possible. However, the decoder can be choked if too
     // many frames are sent at ones. Therefore, limit the interframe delay to
     // |zero_playout_delay_min_pacing_|.
-    int64_t max_wait_time_ms = now_ms >= earliest_next_decode_start_time_
+    int64_t earliest_next_decode_start_time =
+        last_decode_scheduled_ts_ + zero_playout_delay_min_pacing_->ms();
+    int64_t max_wait_time_ms = now_ms >= earliest_next_decode_start_time
                                    ? 0
-                                   : earliest_next_decode_start_time_ - now_ms;
-    earliest_next_decode_start_time_ =
-        now_ms + max_wait_time_ms + zero_playout_delay_min_pacing_->ms();
+                                   : earliest_next_decode_start_time - now_ms;
     return max_wait_time_ms;
   }
   return render_time_ms - now_ms - RequiredDecodeTimeMs() - render_delay_ms_;
diff --git a/modules/video_coding/timing.h b/modules/video_coding/timing.h
index 1583082..2f3fdfe 100644
--- a/modules/video_coding/timing.h
+++ b/modules/video_coding/timing.h
@@ -83,7 +83,7 @@
 
   // Returns the maximum time in ms that we can wait for a frame to become
   // complete before we must pass it to the decoder.
-  virtual int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms);
+  virtual int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) const;
 
   // Returns the current target delay which is required delay + decode time +
   // render delay.
@@ -105,6 +105,9 @@
       absl::optional<int> max_composition_delay_in_frames);
   absl::optional<int> MaxCompositionDelayInFrames() const;
 
+  // Updates the last time a frame was scheduled for decoding.
+  void SetLastDecodeScheduledTimestamp(int64_t last_decode_scheduled_ts);
+
   enum { kDefaultRenderDelayMs = 10 };
   enum { kDelayMaxChangeMsPerS = 100 };
 
@@ -145,10 +148,10 @@
   // used when min playout delay=0 and max playout delay>=0.
   FieldTrialParameter<TimeDelta> zero_playout_delay_min_pacing_
       RTC_GUARDED_BY(mutex_);
-  // An estimate of when the last frame is scheduled to be sent to the decoder.
+  // Timestamp at which the last frame was scheduled to be sent to the decoder.
   // Used only when the RTP header extension playout delay is set to min=0 ms
   // which is indicated by a render time set to 0.
-  int64_t earliest_next_decode_start_time_ RTC_GUARDED_BY(mutex_);
+  int64_t last_decode_scheduled_ts_ RTC_GUARDED_BY(mutex_);
 };
 }  // namespace webrtc
 
diff --git a/modules/video_coding/timing_unittest.cc b/modules/video_coding/timing_unittest.cc
index be6ac52..988e55f 100644
--- a/modules/video_coding/timing_unittest.cc
+++ b/modules/video_coding/timing_unittest.cc
@@ -169,21 +169,26 @@
     clock.AdvanceTimeMilliseconds(kTimeDeltaMs);
     int64_t now_ms = clock.TimeInMilliseconds();
     EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0);
+    timing.SetLastDecodeScheduledTimestamp(now_ms);
   }
   // Another frame submitted at the same time is paced according to the field
   // trial setting.
   int64_t now_ms = clock.TimeInMilliseconds();
   EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs);
-  // If there's a burst of frames, the min pacing interval is summed.
-  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 2 * kMinPacingMs);
-  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 3 * kMinPacingMs);
-  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 4 * kMinPacingMs);
+  // If there's a burst of frames, the wait time is calculated based on next
+  // decode time.
+  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs);
+  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs);
   // Allow a few ms to pass, this should be subtracted from the MaxWaitingTime.
   constexpr int64_t kTwoMs = 2;
   clock.AdvanceTimeMilliseconds(kTwoMs);
   now_ms = clock.TimeInMilliseconds();
   EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms),
-            5 * kMinPacingMs - kTwoMs);
+            kMinPacingMs - kTwoMs);
+  // A frame is decoded at the current time, the wait time should be restored to
+  // pacing delay.
+  timing.SetLastDecodeScheduledTimestamp(now_ms);
+  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs);
 }
 
 TEST(ReceiverTimingTest, DefaultMaxWaitingTimeUnaffectedByPacingExperiment) {