blob: 5c6c79f6f10b25d88c6857e158e25a5208274605 [file] [log] [blame]
stefan@webrtc.org5f284982012-06-28 07:51:161/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Jonas Olssona4d87372019-07-05 17:08:3311#include "video/stream_synchronization.h"
12
kjellander@webrtc.org0fcaf992015-11-26 14:24:5213#include <algorithm>
14
Åsa Persson8fe22fa2019-11-18 13:10:5615#include "system_wrappers/include/clock.h"
Yves Gerey3e707812018-11-28 15:47:4916#include "system_wrappers/include/ntp_time.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3117#include "test/gtest.h"
stefan@webrtc.org5f284982012-06-28 07:51:1618
19namespace webrtc {
Åsa Persson8fe22fa2019-11-18 13:10:5620namespace {
Åsa Persson0e578582020-02-03 10:13:2021constexpr int kMaxChangeMs = 80; // From stream_synchronization.cc
Åsa Persson8fe22fa2019-11-18 13:10:5622constexpr int kDefaultAudioFrequency = 8000;
23constexpr int kDefaultVideoFrequency = 90000;
24constexpr int kSmoothingFilter = 4 * 2;
25} // namespace
stefan@webrtc.org5f284982012-06-28 07:51:1626
27class StreamSynchronizationTest : public ::testing::Test {
Åsa Persson8fe22fa2019-11-18 13:10:5628 public:
29 StreamSynchronizationTest()
30 : sync_(0, 0), clock_sender_(98765000), clock_receiver_(43210000) {}
31
stefan@webrtc.org5f284982012-06-28 07:51:1632 protected:
stefan@webrtc.org7c3523c2012-09-11 07:00:4233 // Generates the necessary RTCP measurements and RTP timestamps and computes
34 // the audio and video delays needed to get the two streams in sync.
Artem Titovab30d722021-07-27 14:22:1135 // `audio_delay_ms` and `video_delay_ms` are the number of milliseconds after
Åsa Persson0e578582020-02-03 10:13:2036 // capture which the frames are received.
Artem Titovab30d722021-07-27 14:22:1137 // `current_audio_delay_ms` is the number of milliseconds which audio is
stefan@webrtc.org7c3523c2012-09-11 07:00:4238 // currently being delayed by the receiver.
39 bool DelayedStreams(int audio_delay_ms,
40 int video_delay_ms,
41 int current_audio_delay_ms,
Åsa Persson0e578582020-02-03 10:13:2042 int* total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:4243 int* total_video_delay_ms) {
Yves Gerey665174f2018-06-19 13:03:0544 int audio_frequency =
45 static_cast<int>(kDefaultAudioFrequency * audio_clock_drift_ + 0.5);
Yves Gerey665174f2018-06-19 13:03:0546 int video_frequency =
47 static_cast<int>(kDefaultVideoFrequency * video_clock_drift_ + 0.5);
Åsa Persson8fe22fa2019-11-18 13:10:5648
49 // Generate NTP/RTP timestamp pair for both streams corresponding to RTCP.
asaperssonde9e5ff2016-11-02 14:14:0350 bool new_sr;
stefan@webrtc.org5f284982012-06-28 07:51:1651 StreamSynchronization::Measurements audio;
52 StreamSynchronization::Measurements video;
Åsa Persson8fe22fa2019-11-18 13:10:5653 NtpTime ntp_time = clock_sender_.CurrentNtpTime();
asaperssonfe50b4d2016-12-22 15:53:5154 uint32_t rtp_timestamp =
Åsa Persson8fe22fa2019-11-18 13:10:5655 clock_sender_.CurrentTime().ms() * audio_frequency / 1000;
asaperssonfe50b4d2016-12-22 15:53:5156 EXPECT_TRUE(audio.rtp_to_ntp.UpdateMeasurements(
57 ntp_time.seconds(), ntp_time.fractions(), rtp_timestamp, &new_sr));
Åsa Persson8fe22fa2019-11-18 13:10:5658 clock_sender_.AdvanceTimeMilliseconds(100);
59 clock_receiver_.AdvanceTimeMilliseconds(100);
60 ntp_time = clock_sender_.CurrentNtpTime();
61 rtp_timestamp = clock_sender_.CurrentTime().ms() * video_frequency / 1000;
asaperssonfe50b4d2016-12-22 15:53:5162 EXPECT_TRUE(video.rtp_to_ntp.UpdateMeasurements(
63 ntp_time.seconds(), ntp_time.fractions(), rtp_timestamp, &new_sr));
Åsa Persson8fe22fa2019-11-18 13:10:5664 clock_sender_.AdvanceTimeMilliseconds(900);
65 clock_receiver_.AdvanceTimeMilliseconds(900);
66 ntp_time = clock_sender_.CurrentNtpTime();
67 rtp_timestamp = clock_sender_.CurrentTime().ms() * audio_frequency / 1000;
asaperssonfe50b4d2016-12-22 15:53:5168 EXPECT_TRUE(audio.rtp_to_ntp.UpdateMeasurements(
69 ntp_time.seconds(), ntp_time.fractions(), rtp_timestamp, &new_sr));
Åsa Persson8fe22fa2019-11-18 13:10:5670 clock_sender_.AdvanceTimeMilliseconds(100);
71 clock_receiver_.AdvanceTimeMilliseconds(100);
72 ntp_time = clock_sender_.CurrentNtpTime();
73 rtp_timestamp = clock_sender_.CurrentTime().ms() * video_frequency / 1000;
asaperssonfe50b4d2016-12-22 15:53:5174 EXPECT_TRUE(video.rtp_to_ntp.UpdateMeasurements(
75 ntp_time.seconds(), ntp_time.fractions(), rtp_timestamp, &new_sr));
Åsa Persson8fe22fa2019-11-18 13:10:5676 clock_sender_.AdvanceTimeMilliseconds(900);
77 clock_receiver_.AdvanceTimeMilliseconds(900);
stefan@webrtc.org5f284982012-06-28 07:51:1678
stefan@webrtc.org7c3523c2012-09-11 07:00:4279 // Capture an audio and a video frame at the same time.
asaperssonb7e7b492016-11-17 10:27:1480 audio.latest_timestamp =
Åsa Persson8fe22fa2019-11-18 13:10:5681 clock_sender_.CurrentTime().ms() * audio_frequency / 1000;
asaperssonb7e7b492016-11-17 10:27:1482 video.latest_timestamp =
Åsa Persson8fe22fa2019-11-18 13:10:5683 clock_sender_.CurrentTime().ms() * video_frequency / 1000;
stefan@webrtc.org5f284982012-06-28 07:51:1684
85 if (audio_delay_ms > video_delay_ms) {
86 // Audio later than video.
Åsa Persson8fe22fa2019-11-18 13:10:5687 clock_receiver_.AdvanceTimeMilliseconds(video_delay_ms);
88 video.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
89 clock_receiver_.AdvanceTimeMilliseconds(audio_delay_ms - video_delay_ms);
90 audio.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
stefan@webrtc.org5f284982012-06-28 07:51:1691 } else {
92 // Video later than audio.
Åsa Persson8fe22fa2019-11-18 13:10:5693 clock_receiver_.AdvanceTimeMilliseconds(audio_delay_ms);
94 audio.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
95 clock_receiver_.AdvanceTimeMilliseconds(video_delay_ms - audio_delay_ms);
96 video.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
stefan@webrtc.org5f284982012-06-28 07:51:1697 }
Åsa Persson0e578582020-02-03 10:13:2098
stefan@webrtc.org7c3523c2012-09-11 07:00:4299 int relative_delay_ms;
Åsa Persson0e578582020-02-03 10:13:20100 EXPECT_TRUE(StreamSynchronization::ComputeRelativeDelay(
101 audio, video, &relative_delay_ms));
stefan@webrtc.org7c3523c2012-09-11 07:00:42102 EXPECT_EQ(video_delay_ms - audio_delay_ms, relative_delay_ms);
Åsa Persson0e578582020-02-03 10:13:20103
Åsa Persson8fe22fa2019-11-18 13:10:56104 return sync_.ComputeDelays(relative_delay_ms, current_audio_delay_ms,
Åsa Persson0e578582020-02-03 10:13:20105 total_audio_delay_ms, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16106 }
107
stefan@webrtc.org7c3523c2012-09-11 07:00:42108 // Simulate audio playback 300 ms after capture and video rendering 100 ms
109 // after capture. Verify that the correct extra delays are calculated for
110 // audio and video, and that they change correctly when we simulate that
111 // NetEQ or the VCM adds more delay to the streams.
Åsa Persson0e578582020-02-03 10:13:20112 void BothDelayedAudioLaterTest(int base_target_delay_ms) {
113 const int kAudioDelayMs = base_target_delay_ms + 300;
114 const int kVideoDelayMs = base_target_delay_ms + 100;
115 int current_audio_delay_ms = base_target_delay_ms;
116 int total_audio_delay_ms = 0;
117 int total_video_delay_ms = base_target_delay_ms;
118 int filtered_move = (kAudioDelayMs - kVideoDelayMs) / kSmoothingFilter;
stefan@webrtc.org7c3523c2012-09-11 07:00:42119
Åsa Persson0e578582020-02-03 10:13:20120 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
121 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42122 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20123 EXPECT_EQ(base_target_delay_ms + filtered_move, total_video_delay_ms);
124 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42125
Åsa Persson0e578582020-02-03 10:13:20126 // Set new current delay.
127 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 13:10:56128 clock_sender_.AdvanceTimeMilliseconds(1000);
129 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 10:13:20130 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
131 // Simulate base_target_delay_ms minimum delay in the VCM.
132 total_video_delay_ms = base_target_delay_ms;
133 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
134 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42135 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20136 EXPECT_EQ(base_target_delay_ms + 2 * filtered_move, total_video_delay_ms);
137 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42138
Åsa Persson0e578582020-02-03 10:13:20139 // Set new current delay.
140 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 13:10:56141 clock_sender_.AdvanceTimeMilliseconds(1000);
142 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 10:13:20143 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
144 // Simulate base_target_delay_ms minimum delay in the VCM.
145 total_video_delay_ms = base_target_delay_ms;
146 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
147 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42148 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20149 EXPECT_EQ(base_target_delay_ms + 3 * filtered_move, total_video_delay_ms);
150 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42151
152 // Simulate that NetEQ introduces some audio delay.
Åsa Persson0e578582020-02-03 10:13:20153 const int kNeteqDelayIncrease = 50;
154 current_audio_delay_ms = base_target_delay_ms + kNeteqDelayIncrease;
Åsa Persson8fe22fa2019-11-18 13:10:56155 clock_sender_.AdvanceTimeMilliseconds(1000);
156 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 10:13:20157 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
158 // Simulate base_target_delay_ms minimum delay in the VCM.
159 total_video_delay_ms = base_target_delay_ms;
160 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
161 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42162 &total_video_delay_ms));
pwestin@webrtc.org63117332013-04-22 18:57:14163 filtered_move = 3 * filtered_move +
Åsa Persson0e578582020-02-03 10:13:20164 (kNeteqDelayIncrease + kAudioDelayMs - kVideoDelayMs) /
Yves Gerey665174f2018-06-19 13:03:05165 kSmoothingFilter;
Åsa Persson0e578582020-02-03 10:13:20166 EXPECT_EQ(base_target_delay_ms + filtered_move, total_video_delay_ms);
167 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42168
169 // Simulate that NetEQ reduces its delay.
Åsa Persson0e578582020-02-03 10:13:20170 const int kNeteqDelayDecrease = 10;
171 current_audio_delay_ms = base_target_delay_ms + kNeteqDelayDecrease;
Åsa Persson8fe22fa2019-11-18 13:10:56172 clock_sender_.AdvanceTimeMilliseconds(1000);
173 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 10:13:20174 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
175 // Simulate base_target_delay_ms minimum delay in the VCM.
176 total_video_delay_ms = base_target_delay_ms;
177 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
178 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42179 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20180 filtered_move =
181 filtered_move + (kNeteqDelayDecrease + kAudioDelayMs - kVideoDelayMs) /
182 kSmoothingFilter;
183 EXPECT_EQ(base_target_delay_ms + filtered_move, total_video_delay_ms);
184 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18185 }
186
Åsa Persson0e578582020-02-03 10:13:20187 void BothDelayedVideoLaterTest(int base_target_delay_ms) {
188 const int kAudioDelayMs = base_target_delay_ms + 100;
189 const int kVideoDelayMs = base_target_delay_ms + 300;
190 int current_audio_delay_ms = base_target_delay_ms;
191 int total_audio_delay_ms = 0;
192 int total_video_delay_ms = base_target_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18193
Åsa Persson0e578582020-02-03 10:13:20194 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
195 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18196 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20197 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
198 // The audio delay is not allowed to change more than this.
199 EXPECT_GE(base_target_delay_ms + kMaxChangeMs, total_audio_delay_ms);
200 int last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18201
Åsa Persson0e578582020-02-03 10:13:20202 // Set new current audio delay.
203 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 13:10:56204 clock_sender_.AdvanceTimeMilliseconds(1000);
205 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20206 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
207 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18208 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20209 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
210 EXPECT_EQ(last_total_audio_delay_ms +
211 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 13:03:05212 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 10:13:20213 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
214 total_audio_delay_ms);
215 last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18216
Åsa Persson0e578582020-02-03 10:13:20217 // Set new current audio delay.
218 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 13:10:56219 clock_sender_.AdvanceTimeMilliseconds(1000);
220 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20221 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
222 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18223 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20224 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
225 EXPECT_EQ(last_total_audio_delay_ms +
226 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 13:03:05227 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 10:13:20228 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
229 total_audio_delay_ms);
230 last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18231
232 // Simulate that NetEQ for some reason reduced the delay.
Åsa Persson0e578582020-02-03 10:13:20233 current_audio_delay_ms = base_target_delay_ms + 10;
Åsa Persson8fe22fa2019-11-18 13:10:56234 clock_sender_.AdvanceTimeMilliseconds(1000);
235 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20236 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
237 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18238 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20239 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
240 EXPECT_EQ(last_total_audio_delay_ms +
241 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 13:03:05242 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 10:13:20243 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
244 total_audio_delay_ms);
245 last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18246
247 // Simulate that NetEQ for some reason significantly increased the delay.
Åsa Persson0e578582020-02-03 10:13:20248 current_audio_delay_ms = base_target_delay_ms + 350;
Åsa Persson8fe22fa2019-11-18 13:10:56249 clock_sender_.AdvanceTimeMilliseconds(1000);
250 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20251 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
252 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18253 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 10:13:20254 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
255 EXPECT_EQ(last_total_audio_delay_ms +
256 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 13:03:05257 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 10:13:20258 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
259 total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42260 }
261
Åsa Persson0e578582020-02-03 10:13:20262 int MaxAudioDelayChangeMs(int current_audio_delay_ms, int delay_ms) const {
263 int diff_ms = (delay_ms - current_audio_delay_ms) / kSmoothingFilter;
264 diff_ms = std::min(diff_ms, kMaxChangeMs);
265 diff_ms = std::max(diff_ms, -kMaxChangeMs);
266 return diff_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16267 }
268
Åsa Persson8fe22fa2019-11-18 13:10:56269 StreamSynchronization sync_;
270 SimulatedClock clock_sender_;
271 SimulatedClock clock_receiver_;
272 double audio_clock_drift_ = 1.0;
273 double video_clock_drift_ = 1.0;
stefan@webrtc.org5f284982012-06-28 07:51:16274};
275
276TEST_F(StreamSynchronizationTest, NoDelay) {
Åsa Persson0e578582020-02-03 10:13:20277 int total_audio_delay_ms = 0;
stefan@webrtc.org5f284982012-06-28 07:51:16278 int total_video_delay_ms = 0;
279
Åsa Persson0e578582020-02-03 10:13:20280 EXPECT_FALSE(DelayedStreams(/*audio_delay_ms=*/0, /*video_delay_ms=*/0,
281 /*current_audio_delay_ms=*/0,
282 &total_audio_delay_ms, &total_video_delay_ms));
283 EXPECT_EQ(0, total_audio_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16284 EXPECT_EQ(0, total_video_delay_ms);
285}
286
Åsa Persson0e578582020-02-03 10:13:20287TEST_F(StreamSynchronizationTest, VideoDelayed) {
288 const int kAudioDelayMs = 200;
289 int total_audio_delay_ms = 0;
stefan@webrtc.org5f284982012-06-28 07:51:16290 int total_video_delay_ms = 0;
291
Åsa Persson0e578582020-02-03 10:13:20292 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, /*video_delay_ms=*/0,
293 /*current_audio_delay_ms=*/0,
294 &total_audio_delay_ms, &total_video_delay_ms));
295 EXPECT_EQ(0, total_audio_delay_ms);
296 // The delay is not allowed to change more than this.
297 EXPECT_EQ(kAudioDelayMs / kSmoothingFilter, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16298
stefan@webrtc.org5f284982012-06-28 07:51:16299 // Simulate 0 minimum delay in the VCM.
300 total_video_delay_ms = 0;
Åsa Persson8fe22fa2019-11-18 13:10:56301 clock_sender_.AdvanceTimeMilliseconds(1000);
302 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20303 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, /*video_delay_ms=*/0,
304 /*current_audio_delay_ms=*/0,
305 &total_audio_delay_ms, &total_video_delay_ms));
306 EXPECT_EQ(0, total_audio_delay_ms);
307 EXPECT_EQ(2 * kAudioDelayMs / kSmoothingFilter, total_video_delay_ms);
308
stefan@webrtc.org5f284982012-06-28 07:51:16309 // Simulate 0 minimum delay in the VCM.
310 total_video_delay_ms = 0;
Åsa Persson0e578582020-02-03 10:13:20311 clock_sender_.AdvanceTimeMilliseconds(1000);
312 clock_receiver_.AdvanceTimeMilliseconds(800);
313 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, /*video_delay_ms=*/0,
314 /*current_audio_delay_ms=*/0,
315 &total_audio_delay_ms, &total_video_delay_ms));
316 EXPECT_EQ(0, total_audio_delay_ms);
317 EXPECT_EQ(3 * kAudioDelayMs / kSmoothingFilter, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16318}
319
Åsa Persson0e578582020-02-03 10:13:20320TEST_F(StreamSynchronizationTest, AudioDelayed) {
321 const int kVideoDelayMs = 200;
stefan@webrtc.org5f284982012-06-28 07:51:16322 int current_audio_delay_ms = 0;
Åsa Persson0e578582020-02-03 10:13:20323 int total_audio_delay_ms = 0;
stefan@webrtc.org5f284982012-06-28 07:51:16324 int total_video_delay_ms = 0;
325
Åsa Persson0e578582020-02-03 10:13:20326 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
327 current_audio_delay_ms, &total_audio_delay_ms,
328 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16329 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 10:13:20330 // The delay is not allowed to change more than this.
331 EXPECT_EQ(kVideoDelayMs / kSmoothingFilter, total_audio_delay_ms);
332 int last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16333
Åsa Persson0e578582020-02-03 10:13:20334 // Set new current audio delay.
335 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 13:10:56336 clock_sender_.AdvanceTimeMilliseconds(1000);
337 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20338 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
339 current_audio_delay_ms, &total_audio_delay_ms,
340 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16341 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 10:13:20342 EXPECT_EQ(last_total_audio_delay_ms +
343 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
344 total_audio_delay_ms);
345 last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16346
Åsa Persson0e578582020-02-03 10:13:20347 // Set new current audio delay.
348 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 13:10:56349 clock_sender_.AdvanceTimeMilliseconds(1000);
350 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20351 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
352 current_audio_delay_ms, &total_audio_delay_ms,
353 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16354 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 10:13:20355 EXPECT_EQ(last_total_audio_delay_ms +
356 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
357 total_audio_delay_ms);
358 last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16359
360 // Simulate that NetEQ for some reason reduced the delay.
pwestin@webrtc.org63117332013-04-22 18:57:14361 current_audio_delay_ms = 10;
Åsa Persson8fe22fa2019-11-18 13:10:56362 clock_sender_.AdvanceTimeMilliseconds(1000);
363 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20364 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
365 current_audio_delay_ms, &total_audio_delay_ms,
366 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16367 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 10:13:20368 EXPECT_EQ(last_total_audio_delay_ms +
369 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
370 total_audio_delay_ms);
371 last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16372
373 // Simulate that NetEQ for some reason significantly increased the delay.
pwestin@webrtc.org63117332013-04-22 18:57:14374 current_audio_delay_ms = 350;
Åsa Persson8fe22fa2019-11-18 13:10:56375 clock_sender_.AdvanceTimeMilliseconds(1000);
376 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 10:13:20377 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
378 current_audio_delay_ms, &total_audio_delay_ms,
379 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16380 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 10:13:20381 EXPECT_EQ(last_total_audio_delay_ms +
382 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
383 total_audio_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16384}
385
Shyam Sadhwani986e7452020-09-14 17:12:54386TEST_F(StreamSynchronizationTest, NoAudioIncomingUnboundedIncrease) {
387 // Test how audio delay can grow unbounded when audio stops coming in.
388 // This is handled in caller of RtpStreamsSynchronizer, for example in
389 // RtpStreamsSynchronizer by not updating delays when audio samples stop
390 // coming in.
391 const int kVideoDelayMs = 300;
392 const int kAudioDelayMs = 100;
393 int current_audio_delay_ms = kAudioDelayMs;
394 int total_audio_delay_ms = 0;
395 int total_video_delay_ms = 0;
396
397 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
398 current_audio_delay_ms, &total_audio_delay_ms,
399 &total_video_delay_ms));
400 EXPECT_EQ(0, total_video_delay_ms);
401 // The delay is not allowed to change more than this.
402 EXPECT_EQ((kVideoDelayMs - kAudioDelayMs) / kSmoothingFilter,
403 total_audio_delay_ms);
404 int last_total_audio_delay_ms = total_audio_delay_ms;
405
406 // Set new current audio delay: simulate audio samples are flowing in.
407 current_audio_delay_ms = total_audio_delay_ms;
408
409 clock_sender_.AdvanceTimeMilliseconds(1000);
410 clock_receiver_.AdvanceTimeMilliseconds(1000);
411 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
412 current_audio_delay_ms, &total_audio_delay_ms,
413 &total_video_delay_ms));
414 EXPECT_EQ(0, total_video_delay_ms);
415 EXPECT_EQ(last_total_audio_delay_ms +
416 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
417 total_audio_delay_ms);
418 last_total_audio_delay_ms = total_audio_delay_ms;
419
420 // Simulate no incoming audio by not update audio delay.
421 const int kSimulationSecs = 300; // 5min
422 const int kMaxDeltaDelayMs = 10000; // max delay for audio in webrtc
423 for (auto time_secs = 0; time_secs < kSimulationSecs; time_secs++) {
424 clock_sender_.AdvanceTimeMilliseconds(1000);
425 clock_receiver_.AdvanceTimeMilliseconds(1000);
426 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
427 current_audio_delay_ms, &total_audio_delay_ms,
428 &total_video_delay_ms));
429 EXPECT_EQ(0, total_video_delay_ms);
430
431 // Audio delay does not go above kMaxDeltaDelayMs.
432 EXPECT_EQ(std::min(kMaxDeltaDelayMs,
433 last_total_audio_delay_ms +
434 MaxAudioDelayChangeMs(current_audio_delay_ms,
435 kVideoDelayMs)),
436 total_audio_delay_ms);
437 last_total_audio_delay_ms = total_audio_delay_ms;
438 }
439 // By now the audio delay has grown unbounded to kMaxDeltaDelayMs.
440 EXPECT_EQ(kMaxDeltaDelayMs, last_total_audio_delay_ms);
441}
442
stefan@webrtc.org5f284982012-06-28 07:51:16443TEST_F(StreamSynchronizationTest, BothDelayedVideoLater) {
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18444 BothDelayedVideoLaterTest(0);
445}
stefan@webrtc.org5f284982012-06-28 07:51:16446
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18447TEST_F(StreamSynchronizationTest, BothDelayedVideoLaterAudioClockDrift) {
448 audio_clock_drift_ = 1.05;
449 BothDelayedVideoLaterTest(0);
450}
stefan@webrtc.org5f284982012-06-28 07:51:16451
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18452TEST_F(StreamSynchronizationTest, BothDelayedVideoLaterVideoClockDrift) {
453 video_clock_drift_ = 1.05;
454 BothDelayedVideoLaterTest(0);
stefan@webrtc.org5f284982012-06-28 07:51:16455}
456
457TEST_F(StreamSynchronizationTest, BothDelayedAudioLater) {
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18458 BothDelayedAudioLaterTest(0);
stefan@webrtc.org7c3523c2012-09-11 07:00:42459}
stefan@webrtc.org5f284982012-06-28 07:51:16460
stefan@webrtc.org7c3523c2012-09-11 07:00:42461TEST_F(StreamSynchronizationTest, BothDelayedAudioClockDrift) {
462 audio_clock_drift_ = 1.05;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18463 BothDelayedAudioLaterTest(0);
stefan@webrtc.org7c3523c2012-09-11 07:00:42464}
stefan@webrtc.org5f284982012-06-28 07:51:16465
stefan@webrtc.org7c3523c2012-09-11 07:00:42466TEST_F(StreamSynchronizationTest, BothDelayedVideoClockDrift) {
467 video_clock_drift_ = 1.05;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18468 BothDelayedAudioLaterTest(0);
stefan@webrtc.org7c3523c2012-09-11 07:00:42469}
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18470
Åsa Persson0e578582020-02-03 10:13:20471TEST_F(StreamSynchronizationTest, BothEquallyDelayed) {
472 const int kDelayMs = 2000;
473 int current_audio_delay_ms = kDelayMs;
474 int total_audio_delay_ms = 0;
475 int total_video_delay_ms = kDelayMs;
476 // In sync, expect no change.
477 EXPECT_FALSE(DelayedStreams(kDelayMs, kDelayMs, current_audio_delay_ms,
478 &total_audio_delay_ms, &total_video_delay_ms));
479 // Trigger another call with the same values, delay should not be modified.
480 total_video_delay_ms = kDelayMs;
481 EXPECT_FALSE(DelayedStreams(kDelayMs, kDelayMs, current_audio_delay_ms,
482 &total_audio_delay_ms, &total_video_delay_ms));
483 // Change delay value, delay should not be modified.
484 const int kDelayMs2 = 5000;
485 current_audio_delay_ms = kDelayMs2;
486 total_video_delay_ms = kDelayMs2;
487 EXPECT_FALSE(DelayedStreams(kDelayMs2, kDelayMs2, current_audio_delay_ms,
488 &total_audio_delay_ms, &total_video_delay_ms));
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18489}
490
491TEST_F(StreamSynchronizationTest, BothDelayedAudioLaterWithBaseDelay) {
Åsa Persson0e578582020-02-03 10:13:20492 const int kBaseTargetDelayMs = 3000;
493 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
494 BothDelayedAudioLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18495}
496
497TEST_F(StreamSynchronizationTest, BothDelayedAudioClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 10:13:20498 const int kBaseTargetDelayMs = 3000;
499 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18500 audio_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 10:13:20501 BothDelayedAudioLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18502}
503
504TEST_F(StreamSynchronizationTest, BothDelayedVideoClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 10:13:20505 const int kBaseTargetDelayMs = 3000;
506 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18507 video_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 10:13:20508 BothDelayedAudioLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18509}
510
511TEST_F(StreamSynchronizationTest, BothDelayedVideoLaterWithBaseDelay) {
Åsa Persson0e578582020-02-03 10:13:20512 const int kBaseTargetDelayMs = 2000;
513 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
514 BothDelayedVideoLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18515}
516
517TEST_F(StreamSynchronizationTest,
518 BothDelayedVideoLaterAudioClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 10:13:20519 const int kBaseTargetDelayMs = 2000;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18520 audio_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 10:13:20521 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
522 BothDelayedVideoLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18523}
524
525TEST_F(StreamSynchronizationTest,
526 BothDelayedVideoLaterVideoClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 10:13:20527 const int kBaseTargetDelayMs = 2000;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18528 video_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 10:13:20529 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
530 BothDelayedVideoLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18531}
532
stefan@webrtc.org5f284982012-06-28 07:51:16533} // namespace webrtc