Calculate pacing delay based on decode start time

Schedule the frames to be decoded based on the pacing delay from the
last decode scheduled time. In the current implementation, multiple
threads and different functions in same thread can call
MaxWaitingTime(), thereby increasing the wait time each time the
function is called. Instead of returning the wait time for a future
frame based on the number of times the function is called, return the
wait time only for the next frame to be decoded. Threads can call the
function repeatedly to check the waiting time for next frame and wake
up and then go back to waiting if an encoded frame is not available.

Change-Id: I00886c1619599f94bde5d5eb87405572e435bd73
Bug: chromium:1237402
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/226502
Reviewed-by: Johannes Kron <[email protected]>
Reviewed-by: Philip Eliasson <[email protected]>
Commit-Queue: Johannes Kron <[email protected]>
Cr-Commit-Position: refs/heads/master@{#34660}

Author: Rezaul Barbhuiya

modules/video_coding/frame_buffer2.cc

@@ -110,6 +110,8 @@ void FrameBuffer::StartWaitForNextFrameOnQueue() {
           if (!frames_to_decode_.empty()) {
             // We have frames, deliver!
             frame = absl::WrapUnique(GetNextFrame());
+            timing_->SetLastDecodeScheduledTimestamp(
+                clock_->TimeInMilliseconds());
           } else if (clock_->TimeInMilliseconds() < latest_return_time_ms_) {
             // If there's no frames to decode and there is still time left, it
             // means that the frame buffer was cleared between creation and

modules/video_coding/frame_buffer2_unittest.cc

@@ -55,7 +55,8 @@ class VCMTimingFake : public VCMTiming {
     return last_ms_;
   }
 
-  int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) override {
+  int64_t MaxWaitingTime(int64_t render_time_ms,
+                         int64_t now_ms) const override {
     return render_time_ms - now_ms - kDecodeTime;
   }
 

modules/video_coding/timing.cc

@@ -35,7 +35,7 @@ VCMTiming::VCMTiming(Clock* clock)
       num_decoded_frames_(0),
       low_latency_renderer_enabled_("enabled", true),
       zero_playout_delay_min_pacing_("min_pacing", TimeDelta::Millis(0)),
-      earliest_next_decode_start_time_(0) {
+      last_decode_scheduled_ts_(0) {
   ParseFieldTrial({&low_latency_renderer_enabled_},
                   field_trial::FindFullName("WebRTC-LowLatencyRenderer"));
   ParseFieldTrial({&zero_playout_delay_min_pacing_},
@@ -171,6 +171,12 @@ int64_t VCMTiming::RenderTimeMs(uint32_t frame_timestamp,
   return RenderTimeMsInternal(frame_timestamp, now_ms);
 }
 
+void VCMTiming::SetLastDecodeScheduledTimestamp(
+    int64_t last_decode_scheduled_ts) {
+  MutexLock lock(&mutex_);
+  last_decode_scheduled_ts_ = last_decode_scheduled_ts;
+}
+
 int64_t VCMTiming::RenderTimeMsInternal(uint32_t frame_timestamp,
                                         int64_t now_ms) const {
   constexpr int kLowLatencyRendererMaxPlayoutDelayMs = 500;
@@ -202,19 +208,20 @@ int VCMTiming::RequiredDecodeTimeMs() const {
   return decode_time_ms;
 }
 
-int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) {
+int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms,
+                                  int64_t now_ms) const {
   MutexLock lock(&mutex_);
 
   if (render_time_ms == 0 && zero_playout_delay_min_pacing_->us() > 0) {
     // |render_time_ms| == 0 indicates that the frame should be decoded and
     // rendered as soon as possible. However, the decoder can be choked if too
     // many frames are sent at ones. Therefore, limit the interframe delay to
     // |zero_playout_delay_min_pacing_|.
-    int64_t max_wait_time_ms = now_ms >= earliest_next_decode_start_time_
+    int64_t earliest_next_decode_start_time =
+        last_decode_scheduled_ts_ + zero_playout_delay_min_pacing_->ms();
+    int64_t max_wait_time_ms = now_ms >= earliest_next_decode_start_time
                                    ? 0
-                                   : earliest_next_decode_start_time_ - now_ms;
-    earliest_next_decode_start_time_ =
-        now_ms + max_wait_time_ms + zero_playout_delay_min_pacing_->ms();
+                                   : earliest_next_decode_start_time - now_ms;
     return max_wait_time_ms;
   }
   return render_time_ms - now_ms - RequiredDecodeTimeMs() - render_delay_ms_;

modules/video_coding/timing.h

@@ -83,7 +83,7 @@ class VCMTiming {
 
   // Returns the maximum time in ms that we can wait for a frame to become
   // complete before we must pass it to the decoder.
-  virtual int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms);
+  virtual int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) const;
 
   // Returns the current target delay which is required delay + decode time +
   // render delay.
@@ -105,6 +105,9 @@ class VCMTiming {
       absl::optional<int> max_composition_delay_in_frames);
   absl::optional<int> MaxCompositionDelayInFrames() const;
 
+  // Updates the last time a frame was scheduled for decoding.
+  void SetLastDecodeScheduledTimestamp(int64_t last_decode_scheduled_ts);
+
   enum { kDefaultRenderDelayMs = 10 };
   enum { kDelayMaxChangeMsPerS = 100 };
 
@@ -145,10 +148,10 @@ class VCMTiming {
   // used when min playout delay=0 and max playout delay>=0.
   FieldTrialParameter<TimeDelta> zero_playout_delay_min_pacing_
       RTC_GUARDED_BY(mutex_);
-  // An estimate of when the last frame is scheduled to be sent to the decoder.
+  // Timestamp at which the last frame was scheduled to be sent to the decoder.
   // Used only when the RTP header extension playout delay is set to min=0 ms
   // which is indicated by a render time set to 0.
-  int64_t earliest_next_decode_start_time_ RTC_GUARDED_BY(mutex_);
+  int64_t last_decode_scheduled_ts_ RTC_GUARDED_BY(mutex_);
 };
 }  // namespace webrtc
 

modules/video_coding/timing_unittest.cc

@@ -169,21 +169,26 @@ TEST(ReceiverTimingTest, MaxWaitingTimeZeroDelayPacingExperiment) {
     clock.AdvanceTimeMilliseconds(kTimeDeltaMs);
     int64_t now_ms = clock.TimeInMilliseconds();
     EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0);
+    timing.SetLastDecodeScheduledTimestamp(now_ms);
   }
   // Another frame submitted at the same time is paced according to the field
   // trial setting.
   int64_t now_ms = clock.TimeInMilliseconds();
   EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs);
-  // If there's a burst of frames, the min pacing interval is summed.
-  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 2 * kMinPacingMs);
-  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 3 * kMinPacingMs);
-  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 4 * kMinPacingMs);
+  // If there's a burst of frames, the wait time is calculated based on next
+  // decode time.
+  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs);
+  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs);
   // Allow a few ms to pass, this should be subtracted from the MaxWaitingTime.
   constexpr int64_t kTwoMs = 2;
   clock.AdvanceTimeMilliseconds(kTwoMs);
   now_ms = clock.TimeInMilliseconds();
   EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms),
-            5 * kMinPacingMs - kTwoMs);
+            kMinPacingMs - kTwoMs);
+  // A frame is decoded at the current time, the wait time should be restored to
+  // pacing delay.
+  timing.SetLastDecodeScheduledTimestamp(now_ms);
+  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs);
 }
 
 TEST(ReceiverTimingTest, DefaultMaxWaitingTimeUnaffectedByPacingExperiment) {