blob: fc9ab32aa2379a4442df548bd004d92f4423c1f0 [file] [log] [blame]
andrew@webrtc.orgb015cbe2012-10-22 18:19:231/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Peter Boströme8f07352015-12-09 11:13:3011#include "webrtc/video/stream_synchronization.h"
andrew@webrtc.orgb015cbe2012-10-22 18:19:2312
pbos@webrtc.org281cff82013-05-17 13:44:4813#include <assert.h>
pbos@webrtc.org3f45c2e2013-08-05 16:22:5314#include <math.h>
stefan@webrtc.orgb1d79312013-11-25 12:03:5615#include <stdlib.h>
pbos@webrtc.org3f45c2e2013-08-05 16:22:5316
17#include <algorithm>
andrew@webrtc.orgb015cbe2012-10-22 18:19:2318
Edward Lemur76de83e2017-07-06 17:44:3419#include "webrtc/rtc_base/logging.h"
andrew@webrtc.orgb015cbe2012-10-22 18:19:2320
21namespace webrtc {
22
pwestin@webrtc.orgf13f1fc2013-04-22 18:57:1423static const int kMaxChangeMs = 80;
24static const int kMaxDeltaDelayMs = 10000;
25static const int kFilterLength = 4;
26// Minimum difference between audio and video to warrant a change.
27static const int kMinDeltaMs = 30;
andrew@webrtc.orgb015cbe2012-10-22 18:19:2328
solenberg9551ae02017-01-31 11:58:4029StreamSynchronization::StreamSynchronization(int video_stream_id,
30 int audio_stream_id)
31 : video_stream_id_(video_stream_id),
32 audio_stream_id_(audio_stream_id),
pwestin@webrtc.orgf13f1fc2013-04-22 18:57:1433 base_target_delay_ms_(0),
Peter Boström220f6652015-05-21 15:00:2434 avg_diff_ms_(0) {
35}
andrew@webrtc.orgb015cbe2012-10-22 18:19:2336
andrew@webrtc.orgb015cbe2012-10-22 18:19:2337bool StreamSynchronization::ComputeRelativeDelay(
38 const Measurements& audio_measurement,
39 const Measurements& video_measurement,
40 int* relative_delay_ms) {
41 assert(relative_delay_ms);
andrew@webrtc.orgb015cbe2012-10-22 18:19:2342 int64_t audio_last_capture_time_ms;
asapersson31fd7852016-12-22 15:53:5143 if (!audio_measurement.rtp_to_ntp.Estimate(audio_measurement.latest_timestamp,
44 &audio_last_capture_time_ms)) {
andrew@webrtc.orgb015cbe2012-10-22 18:19:2345 return false;
46 }
47 int64_t video_last_capture_time_ms;
asapersson31fd7852016-12-22 15:53:5148 if (!video_measurement.rtp_to_ntp.Estimate(video_measurement.latest_timestamp,
49 &video_last_capture_time_ms)) {
andrew@webrtc.orgb015cbe2012-10-22 18:19:2350 return false;
51 }
52 if (video_last_capture_time_ms < 0) {
53 return false;
54 }
55 // Positive diff means that video_measurement is behind audio_measurement.
56 *relative_delay_ms = video_measurement.latest_receive_time_ms -
57 audio_measurement.latest_receive_time_ms -
58 (video_last_capture_time_ms - audio_last_capture_time_ms);
mikhal@webrtc.org9d6fcb32013-02-15 23:22:1859 if (*relative_delay_ms > kMaxDeltaDelayMs ||
60 *relative_delay_ms < -kMaxDeltaDelayMs) {
andrew@webrtc.orgb015cbe2012-10-22 18:19:2361 return false;
62 }
63 return true;
64}
65
66bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
67 int current_audio_delay_ms,
turaj@webrtc.orgd5577342013-05-22 20:39:4368 int* total_audio_delay_target_ms,
andrew@webrtc.orgb015cbe2012-10-22 18:19:2369 int* total_video_delay_target_ms) {
turaj@webrtc.orgd5577342013-05-22 20:39:4370 assert(total_audio_delay_target_ms && total_video_delay_target_ms);
pwestin@webrtc.org4d4ba472013-04-30 16:06:1071
72 int current_video_delay_ms = *total_video_delay_target_ms;
mflodman@webrtc.org022615b2014-04-07 10:56:3173 LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms
mflodman@webrtc.org022615b2014-04-07 10:56:3174 << " current diff: " << relative_delay_ms
solenberg9551ae02017-01-31 11:58:4075 << " for stream " << audio_stream_id_;
andrew@webrtc.orgb015cbe2012-10-22 18:19:2376 // Calculate the difference between the lowest possible video delay and
77 // the current audio delay.
pwestin@webrtc.org4d4ba472013-04-30 16:06:1078 int current_diff_ms = current_video_delay_ms - current_audio_delay_ms +
andrew@webrtc.orgb015cbe2012-10-22 18:19:2379 relative_delay_ms;
80
pwestin@webrtc.orgf13f1fc2013-04-22 18:57:1481 avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ +
82 current_diff_ms) / kFilterLength;
83 if (abs(avg_diff_ms_) < kMinDeltaMs) {
84 // Don't adjust if the diff is within our margin.
85 return false;
86 }
87
88 // Make sure we don't move too fast.
89 int diff_ms = avg_diff_ms_ / 2;
90 diff_ms = std::min(diff_ms, kMaxChangeMs);
91 diff_ms = std::max(diff_ms, -kMaxChangeMs);
92
pwestin@webrtc.org4d4ba472013-04-30 16:06:1093 // Reset the average after a move to prevent overshooting reaction.
94 avg_diff_ms_ = 0;
95
pwestin@webrtc.orgf13f1fc2013-04-22 18:57:1496 if (diff_ms > 0) {
andrew@webrtc.orgb015cbe2012-10-22 18:19:2397 // The minimum video delay is longer than the current audio delay.
pwestin@webrtc.org4d4ba472013-04-30 16:06:1098 // We need to decrease extra video delay, or add extra audio delay.
mflodman46e4b5a2016-08-05 13:28:4599 if (channel_delay_.extra_video_delay_ms > base_target_delay_ms_) {
andrew@webrtc.orgb015cbe2012-10-22 18:19:23100 // We have extra delay added to ViE. Reduce this delay before adding
101 // extra delay to VoE.
mflodman46e4b5a2016-08-05 13:28:45102 channel_delay_.extra_video_delay_ms -= diff_ms;
103 channel_delay_.extra_audio_delay_ms = base_target_delay_ms_;
104 } else { // channel_delay_.extra_video_delay_ms > 0
andrew@webrtc.orgb015cbe2012-10-22 18:19:23105 // We have no extra video delay to remove, increase the audio delay.
mflodman46e4b5a2016-08-05 13:28:45106 channel_delay_.extra_audio_delay_ms += diff_ms;
107 channel_delay_.extra_video_delay_ms = base_target_delay_ms_;
andrew@webrtc.orgb015cbe2012-10-22 18:19:23108 }
pwestin@webrtc.orgf13f1fc2013-04-22 18:57:14109 } else { // if (diff_ms > 0)
pwestin@webrtc.org4d4ba472013-04-30 16:06:10110 // The video delay is lower than the current audio delay.
111 // We need to decrease extra audio delay, or add extra video delay.
mflodman46e4b5a2016-08-05 13:28:45112 if (channel_delay_.extra_audio_delay_ms > base_target_delay_ms_) {
mikhal@webrtc.org9d6fcb32013-02-15 23:22:18113 // We have extra delay in VoiceEngine.
114 // Start with decreasing the voice delay.
pwestin@webrtc.orgf13f1fc2013-04-22 18:57:14115 // Note: diff_ms is negative; add the negative difference.
mflodman46e4b5a2016-08-05 13:28:45116 channel_delay_.extra_audio_delay_ms += diff_ms;
117 channel_delay_.extra_video_delay_ms = base_target_delay_ms_;
118 } else { // channel_delay_.extra_audio_delay_ms > base_target_delay_ms_
andrew@webrtc.orgb015cbe2012-10-22 18:19:23119 // We have no extra delay in VoiceEngine, increase the video delay.
pwestin@webrtc.org4d4ba472013-04-30 16:06:10120 // Note: diff_ms is negative; subtract the negative difference.
mflodman46e4b5a2016-08-05 13:28:45121 channel_delay_.extra_video_delay_ms -= diff_ms; // X - (-Y) = X + Y.
122 channel_delay_.extra_audio_delay_ms = base_target_delay_ms_;
andrew@webrtc.orgb015cbe2012-10-22 18:19:23123 }
124 }
pwestin@webrtc.org4d4ba472013-04-30 16:06:10125
126 // Make sure that video is never below our target.
mflodman46e4b5a2016-08-05 13:28:45127 channel_delay_.extra_video_delay_ms = std::max(
128 channel_delay_.extra_video_delay_ms, base_target_delay_ms_);
pwestin@webrtc.org4d4ba472013-04-30 16:06:10129
130 int new_video_delay_ms;
mflodman46e4b5a2016-08-05 13:28:45131 if (channel_delay_.extra_video_delay_ms > base_target_delay_ms_) {
132 new_video_delay_ms = channel_delay_.extra_video_delay_ms;
pwestin@webrtc.org4d4ba472013-04-30 16:06:10133 } else {
134 // No change to the extra video delay. We are changing audio and we only
135 // allow to change one at the time.
mflodman46e4b5a2016-08-05 13:28:45136 new_video_delay_ms = channel_delay_.last_video_delay_ms;
pwestin@webrtc.org4d4ba472013-04-30 16:06:10137 }
138
139 // Make sure that we don't go below the extra video delay.
140 new_video_delay_ms = std::max(
mflodman46e4b5a2016-08-05 13:28:45141 new_video_delay_ms, channel_delay_.extra_video_delay_ms);
pwestin@webrtc.org4d4ba472013-04-30 16:06:10142
143 // Verify we don't go above the maximum allowed video delay.
144 new_video_delay_ms =
145 std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
146
turaj@webrtc.orgd5577342013-05-22 20:39:43147 int new_audio_delay_ms;
mflodman46e4b5a2016-08-05 13:28:45148 if (channel_delay_.extra_audio_delay_ms > base_target_delay_ms_) {
149 new_audio_delay_ms = channel_delay_.extra_audio_delay_ms;
turaj@webrtc.orgd5577342013-05-22 20:39:43150 } else {
151 // No change to the audio delay. We are changing video and we only
152 // allow to change one at the time.
mflodman46e4b5a2016-08-05 13:28:45153 new_audio_delay_ms = channel_delay_.last_audio_delay_ms;
turaj@webrtc.orgd5577342013-05-22 20:39:43154 }
155
156 // Make sure that we don't go below the extra audio delay.
157 new_audio_delay_ms = std::max(
mflodman46e4b5a2016-08-05 13:28:45158 new_audio_delay_ms, channel_delay_.extra_audio_delay_ms);
pwestin@webrtc.org4d4ba472013-04-30 16:06:10159
160 // Verify we don't go above the maximum allowed audio delay.
turaj@webrtc.orgd5577342013-05-22 20:39:43161 new_audio_delay_ms =
162 std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
pwestin@webrtc.org4d4ba472013-04-30 16:06:10163
turaj@webrtc.orgd5577342013-05-22 20:39:43164 // Remember our last audio and video delays.
mflodman46e4b5a2016-08-05 13:28:45165 channel_delay_.last_video_delay_ms = new_video_delay_ms;
166 channel_delay_.last_audio_delay_ms = new_audio_delay_ms;
pwestin@webrtc.org4d4ba472013-04-30 16:06:10167
mflodman@webrtc.org022615b2014-04-07 10:56:31168 LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms
solenberg9551ae02017-01-31 11:58:40169 << " for video stream " << video_stream_id_
mflodman46e4b5a2016-08-05 13:28:45170 << " and audio delay " << channel_delay_.extra_audio_delay_ms
solenberg9551ae02017-01-31 11:58:40171 << " for audio stream " << audio_stream_id_;
andrew@webrtc.orgb015cbe2012-10-22 18:19:23172
pwestin@webrtc.org4d4ba472013-04-30 16:06:10173 // Return values.
pwestin@webrtc.org4d4ba472013-04-30 16:06:10174 *total_video_delay_target_ms = new_video_delay_ms;
turaj@webrtc.orgd5577342013-05-22 20:39:43175 *total_audio_delay_target_ms = new_audio_delay_ms;
andrew@webrtc.orgb015cbe2012-10-22 18:19:23176 return true;
177}
mikhal@webrtc.org9d6fcb32013-02-15 23:22:18178
179void StreamSynchronization::SetTargetBufferingDelay(int target_delay_ms) {
mikhal@webrtc.org1601d4a2013-02-22 19:30:44180 // Initial extra delay for audio (accounting for existing extra delay).
mflodman46e4b5a2016-08-05 13:28:45181 channel_delay_.extra_audio_delay_ms +=
mikhal@webrtc.org1601d4a2013-02-22 19:30:44182 target_delay_ms - base_target_delay_ms_;
mflodman46e4b5a2016-08-05 13:28:45183 channel_delay_.last_audio_delay_ms +=
turaj@webrtc.orgd5577342013-05-22 20:39:43184 target_delay_ms - base_target_delay_ms_;
pwestin@webrtc.org4d4ba472013-04-30 16:06:10185
mikhal@webrtc.org1601d4a2013-02-22 19:30:44186 // The video delay is compared to the last value (and how much we can update
187 // is limited by that as well).
mflodman46e4b5a2016-08-05 13:28:45188 channel_delay_.last_video_delay_ms +=
mikhal@webrtc.org1601d4a2013-02-22 19:30:44189 target_delay_ms - base_target_delay_ms_;
pwestin@webrtc.org4d4ba472013-04-30 16:06:10190
mflodman46e4b5a2016-08-05 13:28:45191 channel_delay_.extra_video_delay_ms +=
pwestin@webrtc.org4d4ba472013-04-30 16:06:10192 target_delay_ms - base_target_delay_ms_;
193
mikhal@webrtc.org9d6fcb32013-02-15 23:22:18194 // Video is already delayed by the desired amount.
195 base_target_delay_ms_ = target_delay_ms;
mikhal@webrtc.org9d6fcb32013-02-15 23:22:18196}
197
andrew@webrtc.orgb015cbe2012-10-22 18:19:23198} // namespace webrtc