blob: b733a1d2cf0ba01811094f2ea03528c0651b7b37 [file] [log] [blame]
stefan@webrtc.org5f284982012-06-28 07:51:161/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Jonas Olssona4d87372019-07-05 17:08:3311#include "video/stream_synchronization.h"
12
kjellander@webrtc.org0fcaf992015-11-26 14:24:5213#include <algorithm>
14
Åsa Persson8fe22fa2019-11-18 13:10:5615#include "system_wrappers/include/clock.h"
Yves Gerey3e707812018-11-28 15:47:4916#include "system_wrappers/include/ntp_time.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3117#include "test/gtest.h"
stefan@webrtc.org5f284982012-06-28 07:51:1618
19namespace webrtc {
Åsa Persson8fe22fa2019-11-18 13:10:5620namespace {
Åsa Persson0e578582020-02-03 10:13:2021constexpr int kMaxChangeMs = 80; // From stream_synchronization.cc
Åsa Persson8fe22fa2019-11-18 13:10:5622constexpr int kDefaultAudioFrequency = 8000;
23constexpr int kDefaultVideoFrequency = 90000;
24constexpr int kSmoothingFilter = 4 * 2;
25} // namespace
stefan@webrtc.org5f284982012-06-28 07:51:1626
27class StreamSynchronizationTest : public ::testing::Test {
Åsa Persson8fe22fa2019-11-18 13:10:5628 public:
29 StreamSynchronizationTest()
30 : sync_(0, 0), clock_sender_(98765000), clock_receiver_(43210000) {}
31
stefan@webrtc.org5f284982012-06-28 07:51:1632 protected:
stefan@webrtc.org7c3523c2012-09-11 07:00:4233 // Generates the necessary RTCP measurements and RTP timestamps and computes
34 // the audio and video delays needed to get the two streams in sync.
Artem Titovab30d722021-07-27 14:22:1135 // `audio_delay_ms` and `video_delay_ms` are the number of milliseconds after
Åsa Persson0e578582020-02-03 10:13:2036 // capture which the frames are received.
Artem Titovab30d722021-07-27 14:22:1137 // `current_audio_delay_ms` is the number of milliseconds which audio is
stefan@webrtc.org7c3523c2012-09-11 07:00:4238 // currently being delayed by the receiver.
39 bool DelayedStreams(int audio_delay_ms,
40 int video_delay_ms,
41 int current_audio_delay_ms,
Åsa Persson0e578582020-02-03 10:13:2042 int* total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:4243 int* total_video_delay_ms) {
Yves Gerey665174f2018-06-19 13:03:0544 int audio_frequency =
45 static_cast<int>(kDefaultAudioFrequency * audio_clock_drift_ + 0.5);
Yves Gerey665174f2018-06-19 13:03:0546 int video_frequency =
47 static_cast<int>(kDefaultVideoFrequency * video_clock_drift_ + 0.5);
Åsa Persson8fe22fa2019-11-18 13:10:5648
49 // Generate NTP/RTP timestamp pair for both streams corresponding to RTCP.
stefan@webrtc.org5f284982012-06-28 07:51:1650 StreamSynchronization::Measurements audio;
51 StreamSynchronization::Measurements video;
Åsa Persson8fe22fa2019-11-18 13:10:5652 NtpTime ntp_time = clock_sender_.CurrentNtpTime();
asaperssonfe50b4d2016-12-22 15:53:5153 uint32_t rtp_timestamp =
Åsa Persson8fe22fa2019-11-18 13:10:5654 clock_sender_.CurrentTime().ms() * audio_frequency / 1000;
Danil Chapovalovae4fb612022-03-14 11:31:4655 EXPECT_EQ(audio.rtp_to_ntp.UpdateMeasurements(ntp_time, rtp_timestamp),
56 RtpToNtpEstimator::kNewMeasurement);
Åsa Persson8fe22fa2019-11-18 13:10:5657 clock_sender_.AdvanceTimeMilliseconds(100);
58 clock_receiver_.AdvanceTimeMilliseconds(100);
59 ntp_time = clock_sender_.CurrentNtpTime();
60 rtp_timestamp = clock_sender_.CurrentTime().ms() * video_frequency / 1000;
Danil Chapovalovae4fb612022-03-14 11:31:4661 EXPECT_EQ(video.rtp_to_ntp.UpdateMeasurements(ntp_time, rtp_timestamp),
62 RtpToNtpEstimator::kNewMeasurement);
Åsa Persson8fe22fa2019-11-18 13:10:5663 clock_sender_.AdvanceTimeMilliseconds(900);
64 clock_receiver_.AdvanceTimeMilliseconds(900);
65 ntp_time = clock_sender_.CurrentNtpTime();
66 rtp_timestamp = clock_sender_.CurrentTime().ms() * audio_frequency / 1000;
Danil Chapovalovae4fb612022-03-14 11:31:4667 EXPECT_EQ(audio.rtp_to_ntp.UpdateMeasurements(ntp_time, rtp_timestamp),
68 RtpToNtpEstimator::kNewMeasurement);
Åsa Persson8fe22fa2019-11-18 13:10:5669 clock_sender_.AdvanceTimeMilliseconds(100);
70 clock_receiver_.AdvanceTimeMilliseconds(100);
71 ntp_time = clock_sender_.CurrentNtpTime();
72 rtp_timestamp = clock_sender_.CurrentTime().ms() * video_frequency / 1000;
Danil Chapovalovae4fb612022-03-14 11:31:4673 EXPECT_EQ(video.rtp_to_ntp.UpdateMeasurements(ntp_time, rtp_timestamp),
74 RtpToNtpEstimator::kNewMeasurement);
Åsa Persson8fe22fa2019-11-18 13:10:5675 clock_sender_.AdvanceTimeMilliseconds(900);
76 clock_receiver_.AdvanceTimeMilliseconds(900);
stefan@webrtc.org5f284982012-06-28 07:51:1677
stefan@webrtc.org7c3523c2012-09-11 07:00:4278 // Capture an audio and a video frame at the same time.
asaperssonb7e7b492016-11-17 10:27:1479 audio.latest_timestamp =
Åsa Persson8fe22fa2019-11-18 13:10:5680 clock_sender_.CurrentTime().ms() * audio_frequency / 1000;
asaperssonb7e7b492016-11-17 10:27:1481 video.latest_timestamp =
Åsa Persson8fe22fa2019-11-18 13:10:5682 clock_sender_.CurrentTime().ms() * video_frequency / 1000;
stefan@webrtc.org5f284982012-06-28 07:51:1683
84 if (audio_delay_ms > video_delay_ms) {
85 // Audio later than video.
Åsa Persson8fe22fa2019-11-18 13:10:5686 clock_receiver_.AdvanceTimeMilliseconds(video_delay_ms);
87 video.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
88 clock_receiver_.AdvanceTimeMilliseconds(audio_delay_ms - video_delay_ms);
89 audio.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
stefan@webrtc.org5f284982012-06-28 07:51:1690 } else {
91 // Video later than audio.
Åsa Persson8fe22fa2019-11-18 13:10:5692 clock_receiver_.AdvanceTimeMilliseconds(audio_delay_ms);
93 audio.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
94 clock_receiver_.AdvanceTimeMilliseconds(video_delay_ms - audio_delay_ms);
95 video.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
stefan@webrtc.org5f284982012-06-28 07:51:1696 }
Åsa Persson0e578582020-02-03 10:13:2097
stefan@webrtc.org7c3523c2012-09-11 07:00:4298 int relative_delay_ms;
Åsa Persson0e578582020-02-03 10:13:2099 EXPECT_TRUE(StreamSynchronization::ComputeRelativeDelay(
100 audio, video, &relative_delay_ms));
stefan@webrtc.org7c3523c2012-09-11 07:00:42101 EXPECT_EQ(video_delay_ms - audio_delay_ms, relative_delay_ms);
Åsa Persson0e578582020-02-03 10:13:20102
Åsa Persson8fe22fa2019-11-18 13:10:56103 return sync_.ComputeDelays(relative_delay_ms, current_audio_delay_ms,
Åsa Persson0e578582020-02-03 10:13:20104 total_audio_delay_ms, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16105 }
106
stefan@webrtc.org7c3523c2012-09-11 07:00:42107 // Simulate audio playback 300 ms after capture and video rendering 100 ms
108 // after capture. Verify that the correct extra delays are calculated for
109 // audio and video, and that they change correctly when we simulate that
110 // NetEQ or the VCM adds more delay to the streams.
Åsa Persson0e578582020-02-03 10:13:20111 void BothDelayedAudioLaterTest(int base_target_delay_ms) {
112 const int kAudioDelayMs = base_target_delay_ms + 300;
113 const int kVideoDelayMs = base_target_delay_ms + 100;
114 int current_audio_delay_ms = base_target_delay_ms;
115 int total_audio_delay_ms = 0;
116 int total_video_delay_ms = base_target_delay_ms;
117 int filtered_move = (kAudioDelayMs - kVideoDelayMs) / kSmoothingFilter;
stefan@webrtc.org7c3523c2012-09-11 07:00:42118
Åsa Persson0e578582020-02-03 10:13:20119 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
120 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42121 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20122 EXPECT_EQ(base_target_delay_ms + filtered_move, total_video_delay_ms);
123 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42124
Åsa Persson0e578582020-02-03 10:13:20125 // Set new current delay.
126 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 13:10:56127 clock_sender_.AdvanceTimeMilliseconds(1000);
128 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 10:13:20129 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
130 // Simulate base_target_delay_ms minimum delay in the VCM.
131 total_video_delay_ms = base_target_delay_ms;
132 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
133 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42134 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20135 EXPECT_EQ(base_target_delay_ms + 2 * filtered_move, total_video_delay_ms);
136 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42137
Åsa Persson0e578582020-02-03 10:13:20138 // Set new current delay.
139 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 13:10:56140 clock_sender_.AdvanceTimeMilliseconds(1000);
141 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 10:13:20142 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
143 // Simulate base_target_delay_ms minimum delay in the VCM.
144 total_video_delay_ms = base_target_delay_ms;
145 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
146 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42147 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20148 EXPECT_EQ(base_target_delay_ms + 3 * filtered_move, total_video_delay_ms);
149 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42150
151 // Simulate that NetEQ introduces some audio delay.
Åsa Persson0e578582020-02-03 10:13:20152 const int kNeteqDelayIncrease = 50;
153 current_audio_delay_ms = base_target_delay_ms + kNeteqDelayIncrease;
Åsa Persson8fe22fa2019-11-18 13:10:56154 clock_sender_.AdvanceTimeMilliseconds(1000);
155 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 10:13:20156 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
157 // Simulate base_target_delay_ms minimum delay in the VCM.
158 total_video_delay_ms = base_target_delay_ms;
159 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
160 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42161 &total_video_delay_ms));
pwestin@webrtc.org63117332013-04-22 18:57:14162 filtered_move = 3 * filtered_move +
Åsa Persson0e578582020-02-03 10:13:20163 (kNeteqDelayIncrease + kAudioDelayMs - kVideoDelayMs) /
Yves Gerey665174f2018-06-19 13:03:05164 kSmoothingFilter;
Åsa Persson0e578582020-02-03 10:13:20165 EXPECT_EQ(base_target_delay_ms + filtered_move, total_video_delay_ms);
166 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42167
168 // Simulate that NetEQ reduces its delay.
Åsa Persson0e578582020-02-03 10:13:20169 const int kNeteqDelayDecrease = 10;
170 current_audio_delay_ms = base_target_delay_ms + kNeteqDelayDecrease;
Åsa Persson8fe22fa2019-11-18 13:10:56171 clock_sender_.AdvanceTimeMilliseconds(1000);
172 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 10:13:20173 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
174 // Simulate base_target_delay_ms minimum delay in the VCM.
175 total_video_delay_ms = base_target_delay_ms;
176 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
177 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42178 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20179 filtered_move =
180 filtered_move + (kNeteqDelayDecrease + kAudioDelayMs - kVideoDelayMs) /
181 kSmoothingFilter;
182 EXPECT_EQ(base_target_delay_ms + filtered_move, total_video_delay_ms);
183 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18184 }
185
Åsa Persson0e578582020-02-03 10:13:20186 void BothDelayedVideoLaterTest(int base_target_delay_ms) {
187 const int kAudioDelayMs = base_target_delay_ms + 100;
188 const int kVideoDelayMs = base_target_delay_ms + 300;
189 int current_audio_delay_ms = base_target_delay_ms;
190 int total_audio_delay_ms = 0;
191 int total_video_delay_ms = base_target_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18192
Åsa Persson0e578582020-02-03 10:13:20193 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
194 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18195 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20196 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
197 // The audio delay is not allowed to change more than this.
198 EXPECT_GE(base_target_delay_ms + kMaxChangeMs, total_audio_delay_ms);
199 int last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18200
Åsa Persson0e578582020-02-03 10:13:20201 // Set new current audio delay.
202 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 13:10:56203 clock_sender_.AdvanceTimeMilliseconds(1000);
204 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20205 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
206 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18207 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20208 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
209 EXPECT_EQ(last_total_audio_delay_ms +
210 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 13:03:05211 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 10:13:20212 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
213 total_audio_delay_ms);
214 last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18215
Åsa Persson0e578582020-02-03 10:13:20216 // Set new current audio delay.
217 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 13:10:56218 clock_sender_.AdvanceTimeMilliseconds(1000);
219 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20220 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
221 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18222 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20223 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
224 EXPECT_EQ(last_total_audio_delay_ms +
225 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 13:03:05226 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 10:13:20227 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
228 total_audio_delay_ms);
229 last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18230
231 // Simulate that NetEQ for some reason reduced the delay.
Åsa Persson0e578582020-02-03 10:13:20232 current_audio_delay_ms = base_target_delay_ms + 10;
Åsa Persson8fe22fa2019-11-18 13:10:56233 clock_sender_.AdvanceTimeMilliseconds(1000);
234 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20235 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
236 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18237 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20238 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
239 EXPECT_EQ(last_total_audio_delay_ms +
240 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 13:03:05241 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 10:13:20242 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
243 total_audio_delay_ms);
244 last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18245
246 // Simulate that NetEQ for some reason significantly increased the delay.
Åsa Persson0e578582020-02-03 10:13:20247 current_audio_delay_ms = base_target_delay_ms + 350;
Åsa Persson8fe22fa2019-11-18 13:10:56248 clock_sender_.AdvanceTimeMilliseconds(1000);
249 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20250 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
251 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18252 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20253 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
254 EXPECT_EQ(last_total_audio_delay_ms +
255 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 13:03:05256 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 10:13:20257 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
258 total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42259 }
260
Åsa Persson0e578582020-02-03 10:13:20261 int MaxAudioDelayChangeMs(int current_audio_delay_ms, int delay_ms) const {
262 int diff_ms = (delay_ms - current_audio_delay_ms) / kSmoothingFilter;
263 diff_ms = std::min(diff_ms, kMaxChangeMs);
264 diff_ms = std::max(diff_ms, -kMaxChangeMs);
265 return diff_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16266 }
267
Åsa Persson8fe22fa2019-11-18 13:10:56268 StreamSynchronization sync_;
269 SimulatedClock clock_sender_;
270 SimulatedClock clock_receiver_;
271 double audio_clock_drift_ = 1.0;
272 double video_clock_drift_ = 1.0;
stefan@webrtc.org5f284982012-06-28 07:51:16273};
274
275TEST_F(StreamSynchronizationTest, NoDelay) {
Åsa Persson0e578582020-02-03 10:13:20276 int total_audio_delay_ms = 0;
stefan@webrtc.org5f284982012-06-28 07:51:16277 int total_video_delay_ms = 0;
278
Åsa Persson0e578582020-02-03 10:13:20279 EXPECT_FALSE(DelayedStreams(/*audio_delay_ms=*/0, /*video_delay_ms=*/0,
280 /*current_audio_delay_ms=*/0,
281 &total_audio_delay_ms, &total_video_delay_ms));
282 EXPECT_EQ(0, total_audio_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16283 EXPECT_EQ(0, total_video_delay_ms);
284}
285
Åsa Persson0e578582020-02-03 10:13:20286TEST_F(StreamSynchronizationTest, VideoDelayed) {
287 const int kAudioDelayMs = 200;
288 int total_audio_delay_ms = 0;
stefan@webrtc.org5f284982012-06-28 07:51:16289 int total_video_delay_ms = 0;
290
Åsa Persson0e578582020-02-03 10:13:20291 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, /*video_delay_ms=*/0,
292 /*current_audio_delay_ms=*/0,
293 &total_audio_delay_ms, &total_video_delay_ms));
294 EXPECT_EQ(0, total_audio_delay_ms);
295 // The delay is not allowed to change more than this.
296 EXPECT_EQ(kAudioDelayMs / kSmoothingFilter, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16297
stefan@webrtc.org5f284982012-06-28 07:51:16298 // Simulate 0 minimum delay in the VCM.
299 total_video_delay_ms = 0;
Åsa Persson8fe22fa2019-11-18 13:10:56300 clock_sender_.AdvanceTimeMilliseconds(1000);
301 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20302 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, /*video_delay_ms=*/0,
303 /*current_audio_delay_ms=*/0,
304 &total_audio_delay_ms, &total_video_delay_ms));
305 EXPECT_EQ(0, total_audio_delay_ms);
306 EXPECT_EQ(2 * kAudioDelayMs / kSmoothingFilter, total_video_delay_ms);
307
stefan@webrtc.org5f284982012-06-28 07:51:16308 // Simulate 0 minimum delay in the VCM.
309 total_video_delay_ms = 0;
Åsa Persson0e578582020-02-03 10:13:20310 clock_sender_.AdvanceTimeMilliseconds(1000);
311 clock_receiver_.AdvanceTimeMilliseconds(800);
312 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, /*video_delay_ms=*/0,
313 /*current_audio_delay_ms=*/0,
314 &total_audio_delay_ms, &total_video_delay_ms));
315 EXPECT_EQ(0, total_audio_delay_ms);
316 EXPECT_EQ(3 * kAudioDelayMs / kSmoothingFilter, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16317}
318
Åsa Persson0e578582020-02-03 10:13:20319TEST_F(StreamSynchronizationTest, AudioDelayed) {
320 const int kVideoDelayMs = 200;
stefan@webrtc.org5f284982012-06-28 07:51:16321 int current_audio_delay_ms = 0;
Åsa Persson0e578582020-02-03 10:13:20322 int total_audio_delay_ms = 0;
stefan@webrtc.org5f284982012-06-28 07:51:16323 int total_video_delay_ms = 0;
324
Åsa Persson0e578582020-02-03 10:13:20325 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
326 current_audio_delay_ms, &total_audio_delay_ms,
327 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16328 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 10:13:20329 // The delay is not allowed to change more than this.
330 EXPECT_EQ(kVideoDelayMs / kSmoothingFilter, total_audio_delay_ms);
331 int last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16332
Åsa Persson0e578582020-02-03 10:13:20333 // Set new current audio delay.
334 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 13:10:56335 clock_sender_.AdvanceTimeMilliseconds(1000);
336 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20337 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
338 current_audio_delay_ms, &total_audio_delay_ms,
339 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16340 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 10:13:20341 EXPECT_EQ(last_total_audio_delay_ms +
342 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
343 total_audio_delay_ms);
344 last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16345
Åsa Persson0e578582020-02-03 10:13:20346 // Set new current audio delay.
347 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 13:10:56348 clock_sender_.AdvanceTimeMilliseconds(1000);
349 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20350 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
351 current_audio_delay_ms, &total_audio_delay_ms,
352 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16353 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 10:13:20354 EXPECT_EQ(last_total_audio_delay_ms +
355 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
356 total_audio_delay_ms);
357 last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16358
359 // Simulate that NetEQ for some reason reduced the delay.
pwestin@webrtc.org63117332013-04-22 18:57:14360 current_audio_delay_ms = 10;
Åsa Persson8fe22fa2019-11-18 13:10:56361 clock_sender_.AdvanceTimeMilliseconds(1000);
362 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20363 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
364 current_audio_delay_ms, &total_audio_delay_ms,
365 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16366 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 10:13:20367 EXPECT_EQ(last_total_audio_delay_ms +
368 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
369 total_audio_delay_ms);
370 last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16371
372 // Simulate that NetEQ for some reason significantly increased the delay.
pwestin@webrtc.org63117332013-04-22 18:57:14373 current_audio_delay_ms = 350;
Åsa Persson8fe22fa2019-11-18 13:10:56374 clock_sender_.AdvanceTimeMilliseconds(1000);
375 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20376 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
377 current_audio_delay_ms, &total_audio_delay_ms,
378 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16379 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 10:13:20380 EXPECT_EQ(last_total_audio_delay_ms +
381 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
382 total_audio_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16383}
384
Shyam Sadhwani986e7452020-09-14 17:12:54385TEST_F(StreamSynchronizationTest, NoAudioIncomingUnboundedIncrease) {
386 // Test how audio delay can grow unbounded when audio stops coming in.
387 // This is handled in caller of RtpStreamsSynchronizer, for example in
388 // RtpStreamsSynchronizer by not updating delays when audio samples stop
389 // coming in.
390 const int kVideoDelayMs = 300;
391 const int kAudioDelayMs = 100;
392 int current_audio_delay_ms = kAudioDelayMs;
393 int total_audio_delay_ms = 0;
394 int total_video_delay_ms = 0;
395
396 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
397 current_audio_delay_ms, &total_audio_delay_ms,
398 &total_video_delay_ms));
399 EXPECT_EQ(0, total_video_delay_ms);
400 // The delay is not allowed to change more than this.
401 EXPECT_EQ((kVideoDelayMs - kAudioDelayMs) / kSmoothingFilter,
402 total_audio_delay_ms);
403 int last_total_audio_delay_ms = total_audio_delay_ms;
404
405 // Set new current audio delay: simulate audio samples are flowing in.
406 current_audio_delay_ms = total_audio_delay_ms;
407
408 clock_sender_.AdvanceTimeMilliseconds(1000);
409 clock_receiver_.AdvanceTimeMilliseconds(1000);
410 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
411 current_audio_delay_ms, &total_audio_delay_ms,
412 &total_video_delay_ms));
413 EXPECT_EQ(0, total_video_delay_ms);
414 EXPECT_EQ(last_total_audio_delay_ms +
415 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
416 total_audio_delay_ms);
417 last_total_audio_delay_ms = total_audio_delay_ms;
418
419 // Simulate no incoming audio by not update audio delay.
420 const int kSimulationSecs = 300; // 5min
421 const int kMaxDeltaDelayMs = 10000; // max delay for audio in webrtc
422 for (auto time_secs = 0; time_secs < kSimulationSecs; time_secs++) {
423 clock_sender_.AdvanceTimeMilliseconds(1000);
424 clock_receiver_.AdvanceTimeMilliseconds(1000);
425 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
426 current_audio_delay_ms, &total_audio_delay_ms,
427 &total_video_delay_ms));
428 EXPECT_EQ(0, total_video_delay_ms);
429
430 // Audio delay does not go above kMaxDeltaDelayMs.
431 EXPECT_EQ(std::min(kMaxDeltaDelayMs,
432 last_total_audio_delay_ms +
433 MaxAudioDelayChangeMs(current_audio_delay_ms,
434 kVideoDelayMs)),
435 total_audio_delay_ms);
436 last_total_audio_delay_ms = total_audio_delay_ms;
437 }
438 // By now the audio delay has grown unbounded to kMaxDeltaDelayMs.
439 EXPECT_EQ(kMaxDeltaDelayMs, last_total_audio_delay_ms);
440}
441
stefan@webrtc.org5f284982012-06-28 07:51:16442TEST_F(StreamSynchronizationTest, BothDelayedVideoLater) {
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18443 BothDelayedVideoLaterTest(0);
444}
stefan@webrtc.org5f284982012-06-28 07:51:16445
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18446TEST_F(StreamSynchronizationTest, BothDelayedVideoLaterAudioClockDrift) {
447 audio_clock_drift_ = 1.05;
448 BothDelayedVideoLaterTest(0);
449}
stefan@webrtc.org5f284982012-06-28 07:51:16450
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18451TEST_F(StreamSynchronizationTest, BothDelayedVideoLaterVideoClockDrift) {
452 video_clock_drift_ = 1.05;
453 BothDelayedVideoLaterTest(0);
stefan@webrtc.org5f284982012-06-28 07:51:16454}
455
456TEST_F(StreamSynchronizationTest, BothDelayedAudioLater) {
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18457 BothDelayedAudioLaterTest(0);
stefan@webrtc.org7c3523c2012-09-11 07:00:42458}
stefan@webrtc.org5f284982012-06-28 07:51:16459
stefan@webrtc.org7c3523c2012-09-11 07:00:42460TEST_F(StreamSynchronizationTest, BothDelayedAudioClockDrift) {
461 audio_clock_drift_ = 1.05;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18462 BothDelayedAudioLaterTest(0);
stefan@webrtc.org7c3523c2012-09-11 07:00:42463}
stefan@webrtc.org5f284982012-06-28 07:51:16464
stefan@webrtc.org7c3523c2012-09-11 07:00:42465TEST_F(StreamSynchronizationTest, BothDelayedVideoClockDrift) {
466 video_clock_drift_ = 1.05;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18467 BothDelayedAudioLaterTest(0);
stefan@webrtc.org7c3523c2012-09-11 07:00:42468}
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18469
Åsa Persson0e578582020-02-03 10:13:20470TEST_F(StreamSynchronizationTest, BothEquallyDelayed) {
471 const int kDelayMs = 2000;
472 int current_audio_delay_ms = kDelayMs;
473 int total_audio_delay_ms = 0;
474 int total_video_delay_ms = kDelayMs;
475 // In sync, expect no change.
476 EXPECT_FALSE(DelayedStreams(kDelayMs, kDelayMs, current_audio_delay_ms,
477 &total_audio_delay_ms, &total_video_delay_ms));
478 // Trigger another call with the same values, delay should not be modified.
479 total_video_delay_ms = kDelayMs;
480 EXPECT_FALSE(DelayedStreams(kDelayMs, kDelayMs, current_audio_delay_ms,
481 &total_audio_delay_ms, &total_video_delay_ms));
482 // Change delay value, delay should not be modified.
483 const int kDelayMs2 = 5000;
484 current_audio_delay_ms = kDelayMs2;
485 total_video_delay_ms = kDelayMs2;
486 EXPECT_FALSE(DelayedStreams(kDelayMs2, kDelayMs2, current_audio_delay_ms,
487 &total_audio_delay_ms, &total_video_delay_ms));
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18488}
489
490TEST_F(StreamSynchronizationTest, BothDelayedAudioLaterWithBaseDelay) {
Åsa Persson0e578582020-02-03 10:13:20491 const int kBaseTargetDelayMs = 3000;
492 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
493 BothDelayedAudioLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18494}
495
496TEST_F(StreamSynchronizationTest, BothDelayedAudioClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 10:13:20497 const int kBaseTargetDelayMs = 3000;
498 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18499 audio_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 10:13:20500 BothDelayedAudioLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18501}
502
503TEST_F(StreamSynchronizationTest, BothDelayedVideoClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 10:13:20504 const int kBaseTargetDelayMs = 3000;
505 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18506 video_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 10:13:20507 BothDelayedAudioLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18508}
509
510TEST_F(StreamSynchronizationTest, BothDelayedVideoLaterWithBaseDelay) {
Åsa Persson0e578582020-02-03 10:13:20511 const int kBaseTargetDelayMs = 2000;
512 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
513 BothDelayedVideoLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18514}
515
516TEST_F(StreamSynchronizationTest,
517 BothDelayedVideoLaterAudioClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 10:13:20518 const int kBaseTargetDelayMs = 2000;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18519 audio_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 10:13:20520 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
521 BothDelayedVideoLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18522}
523
524TEST_F(StreamSynchronizationTest,
525 BothDelayedVideoLaterVideoClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 10:13:20526 const int kBaseTargetDelayMs = 2000;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18527 video_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 10:13:20528 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
529 BothDelayedVideoLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18530}
531
stefan@webrtc.org5f284982012-06-28 07:51:16532} // namespace webrtc