blob: d86cc792033f5eb8555a96494fef301bc4b79deb [file] [log] [blame]
stefan@webrtc.org5f284982012-06-28 07:51:161/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 04:47:3111#include "video/stream_synchronization.h"
stefan@webrtc.org7c3523c2012-09-11 07:00:4212
stefan@webrtc.org47fadba2013-11-25 12:03:5613#include <stdlib.h>
Jonas Olssona4d87372019-07-05 17:08:3314
pbos@webrtc.org12dc1a32013-08-05 16:22:5315#include <algorithm>
stefan@webrtc.org7c3523c2012-09-11 07:00:4216
Mirko Bonadei92ea95e2017-09-15 04:47:3117#include "rtc_base/logging.h"
stefan@webrtc.org5f284982012-06-28 07:51:1618
19namespace webrtc {
20
pwestin@webrtc.org63117332013-04-22 18:57:1421static const int kMaxChangeMs = 80;
22static const int kMaxDeltaDelayMs = 10000;
23static const int kFilterLength = 4;
24// Minimum difference between audio and video to warrant a change.
25static const int kMinDeltaMs = 30;
stefan@webrtc.org5f284982012-06-28 07:51:1626
Åsa Persson74d2b1d2020-02-10 15:33:2927StreamSynchronization::StreamSynchronization(uint32_t video_stream_id,
28 uint32_t audio_stream_id)
solenberg3ebbcb52017-01-31 11:58:4029 : video_stream_id_(video_stream_id),
30 audio_stream_id_(audio_stream_id),
pwestin@webrtc.org63117332013-04-22 18:57:1431 base_target_delay_ms_(0),
Yves Gerey665174f2018-06-19 13:03:0532 avg_diff_ms_(0) {}
stefan@webrtc.org5f284982012-06-28 07:51:1633
stefan@webrtc.org7c3523c2012-09-11 07:00:4234bool StreamSynchronization::ComputeRelativeDelay(
35 const Measurements& audio_measurement,
36 const Measurements& video_measurement,
37 int* relative_delay_ms) {
Danil Chapovalovae4fb612022-03-14 11:31:4638 NtpTime audio_last_capture_time =
39 audio_measurement.rtp_to_ntp.Estimate(audio_measurement.latest_timestamp);
40 if (!audio_last_capture_time.Valid()) {
stefan@webrtc.org7c3523c2012-09-11 07:00:4241 return false;
42 }
Danil Chapovalovae4fb612022-03-14 11:31:4643 NtpTime video_last_capture_time =
44 video_measurement.rtp_to_ntp.Estimate(video_measurement.latest_timestamp);
45 if (!video_last_capture_time.Valid()) {
stefan@webrtc.org7c3523c2012-09-11 07:00:4246 return false;
47 }
Danil Chapovalovae4fb612022-03-14 11:31:4648 int64_t audio_last_capture_time_ms = audio_last_capture_time.ToMs();
49 int64_t video_last_capture_time_ms = video_last_capture_time.ToMs();
50
stefan@webrtc.org7c3523c2012-09-11 07:00:4251 // Positive diff means that video_measurement is behind audio_measurement.
Yves Gerey665174f2018-06-19 13:03:0552 *relative_delay_ms =
53 video_measurement.latest_receive_time_ms -
stefan@webrtc.org7c3523c2012-09-11 07:00:4254 audio_measurement.latest_receive_time_ms -
55 (video_last_capture_time_ms - audio_last_capture_time_ms);
Åsa Persson74d2b1d2020-02-10 15:33:2956
mikhal@webrtc.orgef9f76a2013-02-15 23:22:1857 if (*relative_delay_ms > kMaxDeltaDelayMs ||
58 *relative_delay_ms < -kMaxDeltaDelayMs) {
stefan@webrtc.org7c3523c2012-09-11 07:00:4259 return false;
60 }
61 return true;
62}
stefan@webrtc.org5f284982012-06-28 07:51:1663
stefan@webrtc.org7c3523c2012-09-11 07:00:4264bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
65 int current_audio_delay_ms,
turaj@webrtc.orge46c8d32013-05-22 20:39:4366 int* total_audio_delay_target_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:4267 int* total_video_delay_target_ms) {
pwestin@webrtc.orgd35964a2013-04-30 16:06:1068 int current_video_delay_ms = *total_video_delay_target_ms;
Åsa Persson4fc52c82019-12-09 11:41:5669
Mirko Bonadei675513b2017-11-09 10:09:2570 RTC_LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms
71 << " current diff: " << relative_delay_ms
72 << " for stream " << audio_stream_id_;
Åsa Persson4fc52c82019-12-09 11:41:5673
74 // Calculate the difference between the lowest possible video delay and the
75 // current audio delay.
Yves Gerey665174f2018-06-19 13:03:0576 int current_diff_ms =
77 current_video_delay_ms - current_audio_delay_ms + relative_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:1678
Yves Gerey665174f2018-06-19 13:03:0579 avg_diff_ms_ =
80 ((kFilterLength - 1) * avg_diff_ms_ + current_diff_ms) / kFilterLength;
pwestin@webrtc.org63117332013-04-22 18:57:1481 if (abs(avg_diff_ms_) < kMinDeltaMs) {
82 // Don't adjust if the diff is within our margin.
83 return false;
84 }
85
86 // Make sure we don't move too fast.
87 int diff_ms = avg_diff_ms_ / 2;
88 diff_ms = std::min(diff_ms, kMaxChangeMs);
89 diff_ms = std::max(diff_ms, -kMaxChangeMs);
90
pwestin@webrtc.orgd35964a2013-04-30 16:06:1091 // Reset the average after a move to prevent overshooting reaction.
92 avg_diff_ms_ = 0;
93
pwestin@webrtc.org63117332013-04-22 18:57:1494 if (diff_ms > 0) {
stefan@webrtc.org5f284982012-06-28 07:51:1695 // The minimum video delay is longer than the current audio delay.
pwestin@webrtc.orgd35964a2013-04-30 16:06:1096 // We need to decrease extra video delay, or add extra audio delay.
Åsa Persson4fc52c82019-12-09 11:41:5697 if (video_delay_.extra_ms > base_target_delay_ms_) {
stefan@webrtc.org5f284982012-06-28 07:51:1698 // We have extra delay added to ViE. Reduce this delay before adding
99 // extra delay to VoE.
Åsa Persson4fc52c82019-12-09 11:41:56100 video_delay_.extra_ms -= diff_ms;
101 audio_delay_.extra_ms = base_target_delay_ms_;
102 } else { // video_delay_.extra_ms > 0
stefan@webrtc.org5f284982012-06-28 07:51:16103 // We have no extra video delay to remove, increase the audio delay.
Åsa Persson4fc52c82019-12-09 11:41:56104 audio_delay_.extra_ms += diff_ms;
105 video_delay_.extra_ms = base_target_delay_ms_;
stefan@webrtc.org5f284982012-06-28 07:51:16106 }
pwestin@webrtc.org63117332013-04-22 18:57:14107 } else { // if (diff_ms > 0)
pwestin@webrtc.orgd35964a2013-04-30 16:06:10108 // The video delay is lower than the current audio delay.
109 // We need to decrease extra audio delay, or add extra video delay.
Åsa Persson4fc52c82019-12-09 11:41:56110 if (audio_delay_.extra_ms > base_target_delay_ms_) {
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18111 // We have extra delay in VoiceEngine.
112 // Start with decreasing the voice delay.
pwestin@webrtc.org63117332013-04-22 18:57:14113 // Note: diff_ms is negative; add the negative difference.
Åsa Persson4fc52c82019-12-09 11:41:56114 audio_delay_.extra_ms += diff_ms;
115 video_delay_.extra_ms = base_target_delay_ms_;
116 } else { // audio_delay_.extra_ms > base_target_delay_ms_
stefan@webrtc.org5f284982012-06-28 07:51:16117 // We have no extra delay in VoiceEngine, increase the video delay.
pwestin@webrtc.orgd35964a2013-04-30 16:06:10118 // Note: diff_ms is negative; subtract the negative difference.
Åsa Persson4fc52c82019-12-09 11:41:56119 video_delay_.extra_ms -= diff_ms; // X - (-Y) = X + Y.
120 audio_delay_.extra_ms = base_target_delay_ms_;
stefan@webrtc.org5f284982012-06-28 07:51:16121 }
122 }
pwestin@webrtc.orgd35964a2013-04-30 16:06:10123
124 // Make sure that video is never below our target.
Åsa Persson4fc52c82019-12-09 11:41:56125 video_delay_.extra_ms =
126 std::max(video_delay_.extra_ms, base_target_delay_ms_);
pwestin@webrtc.orgd35964a2013-04-30 16:06:10127
128 int new_video_delay_ms;
Åsa Persson4fc52c82019-12-09 11:41:56129 if (video_delay_.extra_ms > base_target_delay_ms_) {
130 new_video_delay_ms = video_delay_.extra_ms;
pwestin@webrtc.orgd35964a2013-04-30 16:06:10131 } else {
132 // No change to the extra video delay. We are changing audio and we only
133 // allow to change one at the time.
Åsa Persson4fc52c82019-12-09 11:41:56134 new_video_delay_ms = video_delay_.last_ms;
pwestin@webrtc.orgd35964a2013-04-30 16:06:10135 }
136
137 // Make sure that we don't go below the extra video delay.
Åsa Persson4fc52c82019-12-09 11:41:56138 new_video_delay_ms = std::max(new_video_delay_ms, video_delay_.extra_ms);
pwestin@webrtc.orgd35964a2013-04-30 16:06:10139
140 // Verify we don't go above the maximum allowed video delay.
141 new_video_delay_ms =
142 std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
143
turaj@webrtc.orge46c8d32013-05-22 20:39:43144 int new_audio_delay_ms;
Åsa Persson4fc52c82019-12-09 11:41:56145 if (audio_delay_.extra_ms > base_target_delay_ms_) {
146 new_audio_delay_ms = audio_delay_.extra_ms;
turaj@webrtc.orge46c8d32013-05-22 20:39:43147 } else {
Åsa Persson4fc52c82019-12-09 11:41:56148 // No change to the audio delay. We are changing video and we only allow to
149 // change one at the time.
150 new_audio_delay_ms = audio_delay_.last_ms;
turaj@webrtc.orge46c8d32013-05-22 20:39:43151 }
152
153 // Make sure that we don't go below the extra audio delay.
Åsa Persson4fc52c82019-12-09 11:41:56154 new_audio_delay_ms = std::max(new_audio_delay_ms, audio_delay_.extra_ms);
pwestin@webrtc.orgd35964a2013-04-30 16:06:10155
156 // Verify we don't go above the maximum allowed audio delay.
turaj@webrtc.orge46c8d32013-05-22 20:39:43157 new_audio_delay_ms =
158 std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
pwestin@webrtc.orgd35964a2013-04-30 16:06:10159
Åsa Persson4fc52c82019-12-09 11:41:56160 video_delay_.last_ms = new_video_delay_ms;
161 audio_delay_.last_ms = new_audio_delay_ms;
pwestin@webrtc.orgd35964a2013-04-30 16:06:10162
Mirko Bonadei675513b2017-11-09 10:09:25163 RTC_LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms
164 << " for video stream " << video_stream_id_
Åsa Persson4fc52c82019-12-09 11:41:56165 << " and audio delay " << audio_delay_.extra_ms
Mirko Bonadei675513b2017-11-09 10:09:25166 << " for audio stream " << audio_stream_id_;
stefan@webrtc.org5f284982012-06-28 07:51:16167
pwestin@webrtc.orgd35964a2013-04-30 16:06:10168 *total_video_delay_target_ms = new_video_delay_ms;
turaj@webrtc.orge46c8d32013-05-22 20:39:43169 *total_audio_delay_target_ms = new_audio_delay_ms;
stefan@webrtc.org7c3523c2012-09-11 07:00:42170 return true;
stefan@webrtc.org5f284982012-06-28 07:51:16171}
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18172
173void StreamSynchronization::SetTargetBufferingDelay(int target_delay_ms) {
mikhal@webrtc.org0d8d0102013-02-22 19:30:44174 // Initial extra delay for audio (accounting for existing extra delay).
Åsa Persson4fc52c82019-12-09 11:41:56175 audio_delay_.extra_ms += target_delay_ms - base_target_delay_ms_;
176 audio_delay_.last_ms += target_delay_ms - base_target_delay_ms_;
pwestin@webrtc.orgd35964a2013-04-30 16:06:10177
mikhal@webrtc.org0d8d0102013-02-22 19:30:44178 // The video delay is compared to the last value (and how much we can update
179 // is limited by that as well).
Åsa Persson4fc52c82019-12-09 11:41:56180 video_delay_.last_ms += target_delay_ms - base_target_delay_ms_;
Åsa Persson4fc52c82019-12-09 11:41:56181 video_delay_.extra_ms += target_delay_ms - base_target_delay_ms_;
pwestin@webrtc.orgd35964a2013-04-30 16:06:10182
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18183 // Video is already delayed by the desired amount.
184 base_target_delay_ms_ = target_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18185}
186
Ivo Creusenbef7b052020-09-08 14:30:25187void StreamSynchronization::ReduceAudioDelay() {
188 audio_delay_.extra_ms *= 0.9f;
189}
190
191void StreamSynchronization::ReduceVideoDelay() {
192 video_delay_.extra_ms *= 0.9f;
193}
194
stefan@webrtc.org5f284982012-06-28 07:51:16195} // namespace webrtc