andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Peter Boström | e8f0735 | 2015-12-09 11:13:30 | [diff] [blame] | 11 | #include "webrtc/video/stream_synchronization.h" |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 12 | |
pbos@webrtc.org | 281cff8 | 2013-05-17 13:44:48 | [diff] [blame] | 13 | #include <assert.h> |
pbos@webrtc.org | 3f45c2e | 2013-08-05 16:22:53 | [diff] [blame] | 14 | #include <math.h> |
stefan@webrtc.org | b1d7931 | 2013-11-25 12:03:56 | [diff] [blame] | 15 | #include <stdlib.h> |
pbos@webrtc.org | 3f45c2e | 2013-08-05 16:22:53 | [diff] [blame] | 16 | |
| 17 | #include <algorithm> |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 18 | |
Edward Lemur | 76de83e | 2017-07-06 17:44:34 | [diff] [blame] | 19 | #include "webrtc/rtc_base/logging.h" |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 20 | |
| 21 | namespace webrtc { |
| 22 | |
pwestin@webrtc.org | f13f1fc | 2013-04-22 18:57:14 | [diff] [blame] | 23 | static const int kMaxChangeMs = 80; |
| 24 | static const int kMaxDeltaDelayMs = 10000; |
| 25 | static const int kFilterLength = 4; |
| 26 | // Minimum difference between audio and video to warrant a change. |
| 27 | static const int kMinDeltaMs = 30; |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 28 | |
solenberg | 9551ae0 | 2017-01-31 11:58:40 | [diff] [blame] | 29 | StreamSynchronization::StreamSynchronization(int video_stream_id, |
| 30 | int audio_stream_id) |
| 31 | : video_stream_id_(video_stream_id), |
| 32 | audio_stream_id_(audio_stream_id), |
pwestin@webrtc.org | f13f1fc | 2013-04-22 18:57:14 | [diff] [blame] | 33 | base_target_delay_ms_(0), |
Peter Boström | 220f665 | 2015-05-21 15:00:24 | [diff] [blame] | 34 | avg_diff_ms_(0) { |
| 35 | } |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 36 | |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 37 | bool StreamSynchronization::ComputeRelativeDelay( |
| 38 | const Measurements& audio_measurement, |
| 39 | const Measurements& video_measurement, |
| 40 | int* relative_delay_ms) { |
| 41 | assert(relative_delay_ms); |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 42 | int64_t audio_last_capture_time_ms; |
asapersson | 31fd785 | 2016-12-22 15:53:51 | [diff] [blame] | 43 | if (!audio_measurement.rtp_to_ntp.Estimate(audio_measurement.latest_timestamp, |
| 44 | &audio_last_capture_time_ms)) { |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 45 | return false; |
| 46 | } |
| 47 | int64_t video_last_capture_time_ms; |
asapersson | 31fd785 | 2016-12-22 15:53:51 | [diff] [blame] | 48 | if (!video_measurement.rtp_to_ntp.Estimate(video_measurement.latest_timestamp, |
| 49 | &video_last_capture_time_ms)) { |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 50 | return false; |
| 51 | } |
| 52 | if (video_last_capture_time_ms < 0) { |
| 53 | return false; |
| 54 | } |
| 55 | // Positive diff means that video_measurement is behind audio_measurement. |
| 56 | *relative_delay_ms = video_measurement.latest_receive_time_ms - |
| 57 | audio_measurement.latest_receive_time_ms - |
| 58 | (video_last_capture_time_ms - audio_last_capture_time_ms); |
mikhal@webrtc.org | 9d6fcb3 | 2013-02-15 23:22:18 | [diff] [blame] | 59 | if (*relative_delay_ms > kMaxDeltaDelayMs || |
| 60 | *relative_delay_ms < -kMaxDeltaDelayMs) { |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 61 | return false; |
| 62 | } |
| 63 | return true; |
| 64 | } |
| 65 | |
| 66 | bool StreamSynchronization::ComputeDelays(int relative_delay_ms, |
| 67 | int current_audio_delay_ms, |
turaj@webrtc.org | d557734 | 2013-05-22 20:39:43 | [diff] [blame] | 68 | int* total_audio_delay_target_ms, |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 69 | int* total_video_delay_target_ms) { |
turaj@webrtc.org | d557734 | 2013-05-22 20:39:43 | [diff] [blame] | 70 | assert(total_audio_delay_target_ms && total_video_delay_target_ms); |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 71 | |
| 72 | int current_video_delay_ms = *total_video_delay_target_ms; |
mflodman@webrtc.org | 022615b | 2014-04-07 10:56:31 | [diff] [blame] | 73 | LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms |
mflodman@webrtc.org | 022615b | 2014-04-07 10:56:31 | [diff] [blame] | 74 | << " current diff: " << relative_delay_ms |
solenberg | 9551ae0 | 2017-01-31 11:58:40 | [diff] [blame] | 75 | << " for stream " << audio_stream_id_; |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 76 | // Calculate the difference between the lowest possible video delay and |
| 77 | // the current audio delay. |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 78 | int current_diff_ms = current_video_delay_ms - current_audio_delay_ms + |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 79 | relative_delay_ms; |
| 80 | |
pwestin@webrtc.org | f13f1fc | 2013-04-22 18:57:14 | [diff] [blame] | 81 | avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ + |
| 82 | current_diff_ms) / kFilterLength; |
| 83 | if (abs(avg_diff_ms_) < kMinDeltaMs) { |
| 84 | // Don't adjust if the diff is within our margin. |
| 85 | return false; |
| 86 | } |
| 87 | |
| 88 | // Make sure we don't move too fast. |
| 89 | int diff_ms = avg_diff_ms_ / 2; |
| 90 | diff_ms = std::min(diff_ms, kMaxChangeMs); |
| 91 | diff_ms = std::max(diff_ms, -kMaxChangeMs); |
| 92 | |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 93 | // Reset the average after a move to prevent overshooting reaction. |
| 94 | avg_diff_ms_ = 0; |
| 95 | |
pwestin@webrtc.org | f13f1fc | 2013-04-22 18:57:14 | [diff] [blame] | 96 | if (diff_ms > 0) { |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 97 | // The minimum video delay is longer than the current audio delay. |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 98 | // We need to decrease extra video delay, or add extra audio delay. |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 99 | if (channel_delay_.extra_video_delay_ms > base_target_delay_ms_) { |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 100 | // We have extra delay added to ViE. Reduce this delay before adding |
| 101 | // extra delay to VoE. |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 102 | channel_delay_.extra_video_delay_ms -= diff_ms; |
| 103 | channel_delay_.extra_audio_delay_ms = base_target_delay_ms_; |
| 104 | } else { // channel_delay_.extra_video_delay_ms > 0 |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 105 | // We have no extra video delay to remove, increase the audio delay. |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 106 | channel_delay_.extra_audio_delay_ms += diff_ms; |
| 107 | channel_delay_.extra_video_delay_ms = base_target_delay_ms_; |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 108 | } |
pwestin@webrtc.org | f13f1fc | 2013-04-22 18:57:14 | [diff] [blame] | 109 | } else { // if (diff_ms > 0) |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 110 | // The video delay is lower than the current audio delay. |
| 111 | // We need to decrease extra audio delay, or add extra video delay. |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 112 | if (channel_delay_.extra_audio_delay_ms > base_target_delay_ms_) { |
mikhal@webrtc.org | 9d6fcb3 | 2013-02-15 23:22:18 | [diff] [blame] | 113 | // We have extra delay in VoiceEngine. |
| 114 | // Start with decreasing the voice delay. |
pwestin@webrtc.org | f13f1fc | 2013-04-22 18:57:14 | [diff] [blame] | 115 | // Note: diff_ms is negative; add the negative difference. |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 116 | channel_delay_.extra_audio_delay_ms += diff_ms; |
| 117 | channel_delay_.extra_video_delay_ms = base_target_delay_ms_; |
| 118 | } else { // channel_delay_.extra_audio_delay_ms > base_target_delay_ms_ |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 119 | // We have no extra delay in VoiceEngine, increase the video delay. |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 120 | // Note: diff_ms is negative; subtract the negative difference. |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 121 | channel_delay_.extra_video_delay_ms -= diff_ms; // X - (-Y) = X + Y. |
| 122 | channel_delay_.extra_audio_delay_ms = base_target_delay_ms_; |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 123 | } |
| 124 | } |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 125 | |
| 126 | // Make sure that video is never below our target. |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 127 | channel_delay_.extra_video_delay_ms = std::max( |
| 128 | channel_delay_.extra_video_delay_ms, base_target_delay_ms_); |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 129 | |
| 130 | int new_video_delay_ms; |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 131 | if (channel_delay_.extra_video_delay_ms > base_target_delay_ms_) { |
| 132 | new_video_delay_ms = channel_delay_.extra_video_delay_ms; |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 133 | } else { |
| 134 | // No change to the extra video delay. We are changing audio and we only |
| 135 | // allow to change one at the time. |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 136 | new_video_delay_ms = channel_delay_.last_video_delay_ms; |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 137 | } |
| 138 | |
| 139 | // Make sure that we don't go below the extra video delay. |
| 140 | new_video_delay_ms = std::max( |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 141 | new_video_delay_ms, channel_delay_.extra_video_delay_ms); |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 142 | |
| 143 | // Verify we don't go above the maximum allowed video delay. |
| 144 | new_video_delay_ms = |
| 145 | std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs); |
| 146 | |
turaj@webrtc.org | d557734 | 2013-05-22 20:39:43 | [diff] [blame] | 147 | int new_audio_delay_ms; |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 148 | if (channel_delay_.extra_audio_delay_ms > base_target_delay_ms_) { |
| 149 | new_audio_delay_ms = channel_delay_.extra_audio_delay_ms; |
turaj@webrtc.org | d557734 | 2013-05-22 20:39:43 | [diff] [blame] | 150 | } else { |
| 151 | // No change to the audio delay. We are changing video and we only |
| 152 | // allow to change one at the time. |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 153 | new_audio_delay_ms = channel_delay_.last_audio_delay_ms; |
turaj@webrtc.org | d557734 | 2013-05-22 20:39:43 | [diff] [blame] | 154 | } |
| 155 | |
| 156 | // Make sure that we don't go below the extra audio delay. |
| 157 | new_audio_delay_ms = std::max( |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 158 | new_audio_delay_ms, channel_delay_.extra_audio_delay_ms); |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 159 | |
| 160 | // Verify we don't go above the maximum allowed audio delay. |
turaj@webrtc.org | d557734 | 2013-05-22 20:39:43 | [diff] [blame] | 161 | new_audio_delay_ms = |
| 162 | std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs); |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 163 | |
turaj@webrtc.org | d557734 | 2013-05-22 20:39:43 | [diff] [blame] | 164 | // Remember our last audio and video delays. |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 165 | channel_delay_.last_video_delay_ms = new_video_delay_ms; |
| 166 | channel_delay_.last_audio_delay_ms = new_audio_delay_ms; |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 167 | |
mflodman@webrtc.org | 022615b | 2014-04-07 10:56:31 | [diff] [blame] | 168 | LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms |
solenberg | 9551ae0 | 2017-01-31 11:58:40 | [diff] [blame] | 169 | << " for video stream " << video_stream_id_ |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 170 | << " and audio delay " << channel_delay_.extra_audio_delay_ms |
solenberg | 9551ae0 | 2017-01-31 11:58:40 | [diff] [blame] | 171 | << " for audio stream " << audio_stream_id_; |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 172 | |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 173 | // Return values. |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 174 | *total_video_delay_target_ms = new_video_delay_ms; |
turaj@webrtc.org | d557734 | 2013-05-22 20:39:43 | [diff] [blame] | 175 | *total_audio_delay_target_ms = new_audio_delay_ms; |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 176 | return true; |
| 177 | } |
mikhal@webrtc.org | 9d6fcb3 | 2013-02-15 23:22:18 | [diff] [blame] | 178 | |
| 179 | void StreamSynchronization::SetTargetBufferingDelay(int target_delay_ms) { |
mikhal@webrtc.org | 1601d4a | 2013-02-22 19:30:44 | [diff] [blame] | 180 | // Initial extra delay for audio (accounting for existing extra delay). |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 181 | channel_delay_.extra_audio_delay_ms += |
mikhal@webrtc.org | 1601d4a | 2013-02-22 19:30:44 | [diff] [blame] | 182 | target_delay_ms - base_target_delay_ms_; |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 183 | channel_delay_.last_audio_delay_ms += |
turaj@webrtc.org | d557734 | 2013-05-22 20:39:43 | [diff] [blame] | 184 | target_delay_ms - base_target_delay_ms_; |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 185 | |
mikhal@webrtc.org | 1601d4a | 2013-02-22 19:30:44 | [diff] [blame] | 186 | // The video delay is compared to the last value (and how much we can update |
| 187 | // is limited by that as well). |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 188 | channel_delay_.last_video_delay_ms += |
mikhal@webrtc.org | 1601d4a | 2013-02-22 19:30:44 | [diff] [blame] | 189 | target_delay_ms - base_target_delay_ms_; |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 190 | |
mflodman | 46e4b5a | 2016-08-05 13:28:45 | [diff] [blame] | 191 | channel_delay_.extra_video_delay_ms += |
pwestin@webrtc.org | 4d4ba47 | 2013-04-30 16:06:10 | [diff] [blame] | 192 | target_delay_ms - base_target_delay_ms_; |
| 193 | |
mikhal@webrtc.org | 9d6fcb3 | 2013-02-15 23:22:18 | [diff] [blame] | 194 | // Video is already delayed by the desired amount. |
| 195 | base_target_delay_ms_ = target_delay_ms; |
mikhal@webrtc.org | 9d6fcb3 | 2013-02-15 23:22:18 | [diff] [blame] | 196 | } |
| 197 | |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 | [diff] [blame] | 198 | } // namespace webrtc |