stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
pbos@webrtc.org | f5d4cb1 | 2013-05-17 13:44:48 +0000 | [diff] [blame] | 11 | #include "webrtc/video_engine/stream_synchronization.h" |
stefan@webrtc.org | 7c3523c | 2012-09-11 07:00:42 +0000 | [diff] [blame] | 12 | |
pbos@webrtc.org | f5d4cb1 | 2013-05-17 13:44:48 +0000 | [diff] [blame] | 13 | #include <assert.h> |
pbos@webrtc.org | 12dc1a3 | 2013-08-05 16:22:53 +0000 | [diff] [blame^] | 14 | #include <math.h> |
| 15 | |
| 16 | #include <algorithm> |
stefan@webrtc.org | 7c3523c | 2012-09-11 07:00:42 +0000 | [diff] [blame] | 17 | |
pbos@webrtc.org | f5d4cb1 | 2013-05-17 13:44:48 +0000 | [diff] [blame] | 18 | #include "webrtc/system_wrappers/interface/trace.h" |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 19 | |
| 20 | namespace webrtc { |
| 21 | |
pwestin@webrtc.org | 6311733 | 2013-04-22 18:57:14 +0000 | [diff] [blame] | 22 | static const int kMaxChangeMs = 80; |
| 23 | static const int kMaxDeltaDelayMs = 10000; |
| 24 | static const int kFilterLength = 4; |
| 25 | // Minimum difference between audio and video to warrant a change. |
| 26 | static const int kMinDeltaMs = 30; |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 27 | |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 28 | struct ViESyncDelay { |
| 29 | ViESyncDelay() { |
| 30 | extra_video_delay_ms = 0; |
| 31 | last_video_delay_ms = 0; |
| 32 | extra_audio_delay_ms = 0; |
turaj@webrtc.org | e46c8d3 | 2013-05-22 20:39:43 +0000 | [diff] [blame] | 33 | last_audio_delay_ms = 0; |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 34 | network_delay = 120; |
| 35 | } |
| 36 | |
| 37 | int extra_video_delay_ms; |
| 38 | int last_video_delay_ms; |
| 39 | int extra_audio_delay_ms; |
turaj@webrtc.org | e46c8d3 | 2013-05-22 20:39:43 +0000 | [diff] [blame] | 40 | int last_audio_delay_ms; |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 41 | int network_delay; |
| 42 | }; |
| 43 | |
| 44 | StreamSynchronization::StreamSynchronization(int audio_channel_id, |
| 45 | int video_channel_id) |
| 46 | : channel_delay_(new ViESyncDelay), |
| 47 | audio_channel_id_(audio_channel_id), |
mikhal@webrtc.org | ef9f76a | 2013-02-15 23:22:18 +0000 | [diff] [blame] | 48 | video_channel_id_(video_channel_id), |
pwestin@webrtc.org | 6311733 | 2013-04-22 18:57:14 +0000 | [diff] [blame] | 49 | base_target_delay_ms_(0), |
| 50 | avg_diff_ms_(0) {} |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 51 | |
| 52 | StreamSynchronization::~StreamSynchronization() { |
| 53 | delete channel_delay_; |
| 54 | } |
| 55 | |
stefan@webrtc.org | 7c3523c | 2012-09-11 07:00:42 +0000 | [diff] [blame] | 56 | bool StreamSynchronization::ComputeRelativeDelay( |
| 57 | const Measurements& audio_measurement, |
| 58 | const Measurements& video_measurement, |
| 59 | int* relative_delay_ms) { |
| 60 | assert(relative_delay_ms); |
| 61 | if (audio_measurement.rtcp.size() < 2 || video_measurement.rtcp.size() < 2) { |
| 62 | // We need two RTCP SR reports per stream to do synchronization. |
| 63 | return false; |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 64 | } |
stefan@webrtc.org | 7c3523c | 2012-09-11 07:00:42 +0000 | [diff] [blame] | 65 | int64_t audio_last_capture_time_ms; |
| 66 | if (!synchronization::RtpToNtpMs(audio_measurement.latest_timestamp, |
| 67 | audio_measurement.rtcp, |
| 68 | &audio_last_capture_time_ms)) { |
| 69 | return false; |
| 70 | } |
| 71 | int64_t video_last_capture_time_ms; |
| 72 | if (!synchronization::RtpToNtpMs(video_measurement.latest_timestamp, |
| 73 | video_measurement.rtcp, |
| 74 | &video_last_capture_time_ms)) { |
| 75 | return false; |
| 76 | } |
| 77 | if (video_last_capture_time_ms < 0) { |
| 78 | return false; |
| 79 | } |
| 80 | // Positive diff means that video_measurement is behind audio_measurement. |
| 81 | *relative_delay_ms = video_measurement.latest_receive_time_ms - |
| 82 | audio_measurement.latest_receive_time_ms - |
| 83 | (video_last_capture_time_ms - audio_last_capture_time_ms); |
mikhal@webrtc.org | ef9f76a | 2013-02-15 23:22:18 +0000 | [diff] [blame] | 84 | if (*relative_delay_ms > kMaxDeltaDelayMs || |
| 85 | *relative_delay_ms < -kMaxDeltaDelayMs) { |
stefan@webrtc.org | 7c3523c | 2012-09-11 07:00:42 +0000 | [diff] [blame] | 86 | return false; |
| 87 | } |
| 88 | return true; |
| 89 | } |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 90 | |
stefan@webrtc.org | 7c3523c | 2012-09-11 07:00:42 +0000 | [diff] [blame] | 91 | bool StreamSynchronization::ComputeDelays(int relative_delay_ms, |
| 92 | int current_audio_delay_ms, |
turaj@webrtc.org | e46c8d3 | 2013-05-22 20:39:43 +0000 | [diff] [blame] | 93 | int* total_audio_delay_target_ms, |
stefan@webrtc.org | 7c3523c | 2012-09-11 07:00:42 +0000 | [diff] [blame] | 94 | int* total_video_delay_target_ms) { |
turaj@webrtc.org | e46c8d3 | 2013-05-22 20:39:43 +0000 | [diff] [blame] | 95 | assert(total_audio_delay_target_ms && total_video_delay_target_ms); |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 96 | |
| 97 | int current_video_delay_ms = *total_video_delay_target_ms; |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 98 | WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_, |
| 99 | "Audio delay is: %d for voice channel: %d", |
| 100 | current_audio_delay_ms, audio_channel_id_); |
| 101 | WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_, |
| 102 | "Network delay diff is: %d for voice channel: %d", |
| 103 | channel_delay_->network_delay, audio_channel_id_); |
| 104 | // Calculate the difference between the lowest possible video delay and |
| 105 | // the current audio delay. |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 106 | WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_, |
| 107 | "Current diff is: %d for audio channel: %d", |
stefan@webrtc.org | 7c3523c | 2012-09-11 07:00:42 +0000 | [diff] [blame] | 108 | relative_delay_ms, audio_channel_id_); |
pwestin@webrtc.org | 6311733 | 2013-04-22 18:57:14 +0000 | [diff] [blame] | 109 | |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 110 | int current_diff_ms = current_video_delay_ms - current_audio_delay_ms + |
stefan@webrtc.org | 7c3523c | 2012-09-11 07:00:42 +0000 | [diff] [blame] | 111 | relative_delay_ms; |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 112 | |
pwestin@webrtc.org | 6311733 | 2013-04-22 18:57:14 +0000 | [diff] [blame] | 113 | avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ + |
| 114 | current_diff_ms) / kFilterLength; |
| 115 | if (abs(avg_diff_ms_) < kMinDeltaMs) { |
| 116 | // Don't adjust if the diff is within our margin. |
| 117 | return false; |
| 118 | } |
| 119 | |
| 120 | // Make sure we don't move too fast. |
| 121 | int diff_ms = avg_diff_ms_ / 2; |
| 122 | diff_ms = std::min(diff_ms, kMaxChangeMs); |
| 123 | diff_ms = std::max(diff_ms, -kMaxChangeMs); |
| 124 | |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 125 | // Reset the average after a move to prevent overshooting reaction. |
| 126 | avg_diff_ms_ = 0; |
| 127 | |
pwestin@webrtc.org | 6311733 | 2013-04-22 18:57:14 +0000 | [diff] [blame] | 128 | if (diff_ms > 0) { |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 129 | // The minimum video delay is longer than the current audio delay. |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 130 | // We need to decrease extra video delay, or add extra audio delay. |
| 131 | if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) { |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 132 | // We have extra delay added to ViE. Reduce this delay before adding |
| 133 | // extra delay to VoE. |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 134 | channel_delay_->extra_video_delay_ms -= diff_ms; |
mikhal@webrtc.org | ef9f76a | 2013-02-15 23:22:18 +0000 | [diff] [blame] | 135 | channel_delay_->extra_audio_delay_ms = base_target_delay_ms_; |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 136 | } else { // channel_delay_->extra_video_delay_ms > 0 |
| 137 | // We have no extra video delay to remove, increase the audio delay. |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 138 | channel_delay_->extra_audio_delay_ms += diff_ms; |
| 139 | channel_delay_->extra_video_delay_ms = base_target_delay_ms_; |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 140 | } |
pwestin@webrtc.org | 6311733 | 2013-04-22 18:57:14 +0000 | [diff] [blame] | 141 | } else { // if (diff_ms > 0) |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 142 | // The video delay is lower than the current audio delay. |
| 143 | // We need to decrease extra audio delay, or add extra video delay. |
mikhal@webrtc.org | ef9f76a | 2013-02-15 23:22:18 +0000 | [diff] [blame] | 144 | if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) { |
| 145 | // We have extra delay in VoiceEngine. |
| 146 | // Start with decreasing the voice delay. |
pwestin@webrtc.org | 6311733 | 2013-04-22 18:57:14 +0000 | [diff] [blame] | 147 | // Note: diff_ms is negative; add the negative difference. |
| 148 | channel_delay_->extra_audio_delay_ms += diff_ms; |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 149 | channel_delay_->extra_video_delay_ms = base_target_delay_ms_; |
| 150 | } else { // channel_delay_->extra_audio_delay_ms > base_target_delay_ms_ |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 151 | // We have no extra delay in VoiceEngine, increase the video delay. |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 152 | // Note: diff_ms is negative; subtract the negative difference. |
| 153 | channel_delay_->extra_video_delay_ms -= diff_ms; // X - (-Y) = X + Y. |
mikhal@webrtc.org | ef9f76a | 2013-02-15 23:22:18 +0000 | [diff] [blame] | 154 | channel_delay_->extra_audio_delay_ms = base_target_delay_ms_; |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 155 | } |
| 156 | } |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 157 | |
| 158 | // Make sure that video is never below our target. |
| 159 | channel_delay_->extra_video_delay_ms = std::max( |
| 160 | channel_delay_->extra_video_delay_ms, base_target_delay_ms_); |
| 161 | |
| 162 | int new_video_delay_ms; |
| 163 | if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) { |
| 164 | new_video_delay_ms = channel_delay_->extra_video_delay_ms; |
| 165 | } else { |
| 166 | // No change to the extra video delay. We are changing audio and we only |
| 167 | // allow to change one at the time. |
| 168 | new_video_delay_ms = channel_delay_->last_video_delay_ms; |
| 169 | } |
| 170 | |
| 171 | // Make sure that we don't go below the extra video delay. |
| 172 | new_video_delay_ms = std::max( |
| 173 | new_video_delay_ms, channel_delay_->extra_video_delay_ms); |
| 174 | |
| 175 | // Verify we don't go above the maximum allowed video delay. |
| 176 | new_video_delay_ms = |
| 177 | std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs); |
| 178 | |
turaj@webrtc.org | e46c8d3 | 2013-05-22 20:39:43 +0000 | [diff] [blame] | 179 | int new_audio_delay_ms; |
| 180 | if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) { |
| 181 | new_audio_delay_ms = channel_delay_->extra_audio_delay_ms; |
| 182 | } else { |
| 183 | // No change to the audio delay. We are changing video and we only |
| 184 | // allow to change one at the time. |
| 185 | new_audio_delay_ms = channel_delay_->last_audio_delay_ms; |
| 186 | } |
| 187 | |
| 188 | // Make sure that we don't go below the extra audio delay. |
| 189 | new_audio_delay_ms = std::max( |
| 190 | new_audio_delay_ms, channel_delay_->extra_audio_delay_ms); |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 191 | |
| 192 | // Verify we don't go above the maximum allowed audio delay. |
turaj@webrtc.org | e46c8d3 | 2013-05-22 20:39:43 +0000 | [diff] [blame] | 193 | new_audio_delay_ms = |
| 194 | std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs); |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 195 | |
turaj@webrtc.org | e46c8d3 | 2013-05-22 20:39:43 +0000 | [diff] [blame] | 196 | // Remember our last audio and video delays. |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 197 | channel_delay_->last_video_delay_ms = new_video_delay_ms; |
turaj@webrtc.org | e46c8d3 | 2013-05-22 20:39:43 +0000 | [diff] [blame] | 198 | channel_delay_->last_audio_delay_ms = new_audio_delay_ms; |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 199 | |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 200 | WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_, |
| 201 | "Sync video delay %d ms for video channel and audio delay %d for audio " |
| 202 | "channel %d", |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 203 | new_video_delay_ms, channel_delay_->extra_audio_delay_ms, |
| 204 | audio_channel_id_); |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 205 | |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 206 | // Return values. |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 207 | *total_video_delay_target_ms = new_video_delay_ms; |
turaj@webrtc.org | e46c8d3 | 2013-05-22 20:39:43 +0000 | [diff] [blame] | 208 | *total_audio_delay_target_ms = new_audio_delay_ms; |
stefan@webrtc.org | 7c3523c | 2012-09-11 07:00:42 +0000 | [diff] [blame] | 209 | return true; |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 210 | } |
mikhal@webrtc.org | ef9f76a | 2013-02-15 23:22:18 +0000 | [diff] [blame] | 211 | |
| 212 | void StreamSynchronization::SetTargetBufferingDelay(int target_delay_ms) { |
mikhal@webrtc.org | 0d8d010 | 2013-02-22 19:30:44 +0000 | [diff] [blame] | 213 | // Initial extra delay for audio (accounting for existing extra delay). |
| 214 | channel_delay_->extra_audio_delay_ms += |
| 215 | target_delay_ms - base_target_delay_ms_; |
turaj@webrtc.org | e46c8d3 | 2013-05-22 20:39:43 +0000 | [diff] [blame] | 216 | channel_delay_->last_audio_delay_ms += |
| 217 | target_delay_ms - base_target_delay_ms_; |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 218 | |
mikhal@webrtc.org | 0d8d010 | 2013-02-22 19:30:44 +0000 | [diff] [blame] | 219 | // The video delay is compared to the last value (and how much we can update |
| 220 | // is limited by that as well). |
| 221 | channel_delay_->last_video_delay_ms += |
| 222 | target_delay_ms - base_target_delay_ms_; |
pwestin@webrtc.org | d35964a | 2013-04-30 16:06:10 +0000 | [diff] [blame] | 223 | |
| 224 | channel_delay_->extra_video_delay_ms += |
| 225 | target_delay_ms - base_target_delay_ms_; |
| 226 | |
mikhal@webrtc.org | ef9f76a | 2013-02-15 23:22:18 +0000 | [diff] [blame] | 227 | // Video is already delayed by the desired amount. |
| 228 | base_target_delay_ms_ = target_delay_ms; |
mikhal@webrtc.org | ef9f76a | 2013-02-15 23:22:18 +0000 | [diff] [blame] | 229 | } |
| 230 | |
stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 231 | } // namespace webrtc |