stefan@webrtc.org | 5f28498 | 2012-06-28 07:51:16 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
| 11 | #include "video_engine/stream_synchronization.h" |
| 12 | #include "system_wrappers/interface/trace.h" |
| 13 | |
| 14 | namespace webrtc { |
| 15 | |
| 16 | enum { kMaxVideoDiffMs = 80 }; |
| 17 | enum { kMaxAudioDiffMs = 80 }; |
| 18 | enum { kMaxDelay = 1500 }; |
| 19 | |
| 20 | const float FracMS = 4.294967296E6f; |
| 21 | |
| 22 | struct ViESyncDelay { |
| 23 | ViESyncDelay() { |
| 24 | extra_video_delay_ms = 0; |
| 25 | last_video_delay_ms = 0; |
| 26 | extra_audio_delay_ms = 0; |
| 27 | last_sync_delay = 0; |
| 28 | network_delay = 120; |
| 29 | } |
| 30 | |
| 31 | int extra_video_delay_ms; |
| 32 | int last_video_delay_ms; |
| 33 | int extra_audio_delay_ms; |
| 34 | int last_sync_delay; |
| 35 | int network_delay; |
| 36 | }; |
| 37 | |
| 38 | StreamSynchronization::StreamSynchronization(int audio_channel_id, |
| 39 | int video_channel_id) |
| 40 | : channel_delay_(new ViESyncDelay), |
| 41 | audio_channel_id_(audio_channel_id), |
| 42 | video_channel_id_(video_channel_id) {} |
| 43 | |
| 44 | StreamSynchronization::~StreamSynchronization() { |
| 45 | delete channel_delay_; |
| 46 | } |
| 47 | |
| 48 | int StreamSynchronization::ComputeDelays(const Measurements& audio, |
| 49 | int current_audio_delay_ms, |
| 50 | int* extra_audio_delay_ms, |
| 51 | const Measurements& video, |
| 52 | int* total_video_delay_target_ms) { |
| 53 | // ReceivedNTPxxx is NTP at sender side when sent. |
| 54 | // RTCPArrivalTimexxx is NTP at receiver side when received. |
| 55 | // can't use ConvertNTPTimeToMS since calculation can be |
| 56 | // negative |
| 57 | int NTPdiff = (audio.received_ntp_secs - video.received_ntp_secs) |
| 58 | * 1000; // ms |
| 59 | float ntp_diff_frac = audio.received_ntp_frac / FracMS - |
| 60 | video.received_ntp_frac / FracMS; |
| 61 | if (ntp_diff_frac > 0.0f) |
| 62 | NTPdiff += static_cast<int>(ntp_diff_frac + 0.5f); |
| 63 | else |
| 64 | NTPdiff += static_cast<int>(ntp_diff_frac - 0.5f); |
| 65 | |
| 66 | int RTCPdiff = (audio.rtcp_arrivaltime_secs - video.rtcp_arrivaltime_secs) |
| 67 | * 1000; // ms |
| 68 | float rtcp_diff_frac = audio.rtcp_arrivaltime_frac / FracMS - |
| 69 | video.rtcp_arrivaltime_frac / FracMS; |
| 70 | if (rtcp_diff_frac > 0.0f) |
| 71 | RTCPdiff += static_cast<int>(rtcp_diff_frac + 0.5f); |
| 72 | else |
| 73 | RTCPdiff += static_cast<int>(rtcp_diff_frac - 0.5f); |
| 74 | |
| 75 | int diff = NTPdiff - RTCPdiff; |
| 76 | // if diff is + video is behind |
| 77 | if (diff < -1000 || diff > 1000) { |
| 78 | // unresonable ignore value. |
| 79 | return -1; |
| 80 | } |
| 81 | channel_delay_->network_delay = diff; |
| 82 | |
| 83 | WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_, |
| 84 | "Audio delay is: %d for voice channel: %d", |
| 85 | current_audio_delay_ms, audio_channel_id_); |
| 86 | WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_, |
| 87 | "Network delay diff is: %d for voice channel: %d", |
| 88 | channel_delay_->network_delay, audio_channel_id_); |
| 89 | // Calculate the difference between the lowest possible video delay and |
| 90 | // the current audio delay. |
| 91 | int current_diff_ms = *total_video_delay_target_ms - current_audio_delay_ms + |
| 92 | channel_delay_->network_delay; |
| 93 | WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_, |
| 94 | "Current diff is: %d for audio channel: %d", |
| 95 | current_diff_ms, audio_channel_id_); |
| 96 | |
| 97 | int video_delay_ms = 0; |
| 98 | if (current_diff_ms > 0) { |
| 99 | // The minimum video delay is longer than the current audio delay. |
| 100 | // We need to decrease extra video delay, if we have added extra delay |
| 101 | // earlier, or add extra audio delay. |
| 102 | if (channel_delay_->extra_video_delay_ms > 0) { |
| 103 | // We have extra delay added to ViE. Reduce this delay before adding |
| 104 | // extra delay to VoE. |
| 105 | |
| 106 | // This is the desired delay, we can't reduce more than this. |
| 107 | video_delay_ms = *total_video_delay_target_ms; |
| 108 | |
| 109 | // Check that we don't reduce the delay more than what is allowed. |
| 110 | if (video_delay_ms < |
| 111 | channel_delay_->last_video_delay_ms - kMaxVideoDiffMs) { |
| 112 | video_delay_ms = |
| 113 | channel_delay_->last_video_delay_ms - kMaxVideoDiffMs; |
| 114 | channel_delay_->extra_video_delay_ms = |
| 115 | video_delay_ms - *total_video_delay_target_ms; |
| 116 | } else { |
| 117 | channel_delay_->extra_video_delay_ms = 0; |
| 118 | } |
| 119 | channel_delay_->last_video_delay_ms = video_delay_ms; |
| 120 | channel_delay_->last_sync_delay = -1; |
| 121 | channel_delay_->extra_audio_delay_ms = 0; |
| 122 | } else { // channel_delay_->extra_video_delay_ms > 0 |
| 123 | // We have no extra video delay to remove, increase the audio delay. |
| 124 | if (channel_delay_->last_sync_delay >= 0) { |
| 125 | // We have increased the audio delay earlier, increase it even more. |
| 126 | int audio_diff_ms = current_diff_ms / 2; |
| 127 | if (audio_diff_ms > kMaxAudioDiffMs) { |
| 128 | // We only allow a maximum change of KMaxAudioDiffMS for audio |
| 129 | // due to NetEQ maximum changes. |
| 130 | audio_diff_ms = kMaxAudioDiffMs; |
| 131 | } |
| 132 | // Increase the audio delay |
| 133 | channel_delay_->extra_audio_delay_ms += audio_diff_ms; |
| 134 | |
| 135 | // Don't set a too high delay. |
| 136 | if (channel_delay_->extra_audio_delay_ms > kMaxDelay) { |
| 137 | channel_delay_->extra_audio_delay_ms = kMaxDelay; |
| 138 | } |
| 139 | |
| 140 | // Don't add any extra video delay. |
| 141 | video_delay_ms = *total_video_delay_target_ms; |
| 142 | channel_delay_->extra_video_delay_ms = 0; |
| 143 | channel_delay_->last_video_delay_ms = video_delay_ms; |
| 144 | channel_delay_->last_sync_delay = 1; |
| 145 | } else { // channel_delay_->last_sync_delay >= 0 |
| 146 | // First time after a delay change, don't add any extra delay. |
| 147 | // This is to not toggle back and forth too much. |
| 148 | channel_delay_->extra_audio_delay_ms = 0; |
| 149 | // Set minimum video delay |
| 150 | video_delay_ms = *total_video_delay_target_ms; |
| 151 | channel_delay_->extra_video_delay_ms = 0; |
| 152 | channel_delay_->last_video_delay_ms = video_delay_ms; |
| 153 | channel_delay_->last_sync_delay = 0; |
| 154 | } |
| 155 | } |
| 156 | } else { // if (current_diffMS > 0) |
| 157 | // The minimum video delay is lower than the current audio delay. |
| 158 | // We need to decrease possible extra audio delay, or |
| 159 | // add extra video delay. |
| 160 | |
| 161 | if (channel_delay_->extra_audio_delay_ms > 0) { |
| 162 | // We have extra delay in VoiceEngine |
| 163 | // Start with decreasing the voice delay |
| 164 | int audio_diff_ms = current_diff_ms / 2; |
| 165 | if (audio_diff_ms < -1 * kMaxAudioDiffMs) { |
| 166 | // Don't change the delay too much at once. |
| 167 | audio_diff_ms = -1 * kMaxAudioDiffMs; |
| 168 | } |
| 169 | // Add the negative difference. |
| 170 | channel_delay_->extra_audio_delay_ms += audio_diff_ms; |
| 171 | |
| 172 | if (channel_delay_->extra_audio_delay_ms < 0) { |
| 173 | // Negative values not allowed. |
| 174 | channel_delay_->extra_audio_delay_ms = 0; |
| 175 | channel_delay_->last_sync_delay = 0; |
| 176 | } else { |
| 177 | // There is more audio delay to use for the next round. |
| 178 | channel_delay_->last_sync_delay = 1; |
| 179 | } |
| 180 | |
| 181 | // Keep the video delay at the minimum values. |
| 182 | video_delay_ms = *total_video_delay_target_ms; |
| 183 | channel_delay_->extra_video_delay_ms = 0; |
| 184 | channel_delay_->last_video_delay_ms = video_delay_ms; |
| 185 | } else { // channel_delay_->extra_audio_delay_ms > 0 |
| 186 | // We have no extra delay in VoiceEngine, increase the video delay. |
| 187 | channel_delay_->extra_audio_delay_ms = 0; |
| 188 | |
| 189 | // Make the difference positive. |
| 190 | int video_diff_ms = -1 * current_diff_ms; |
| 191 | |
| 192 | // This is the desired delay. |
| 193 | video_delay_ms = *total_video_delay_target_ms + video_diff_ms; |
| 194 | if (video_delay_ms > channel_delay_->last_video_delay_ms) { |
| 195 | if (video_delay_ms > |
| 196 | channel_delay_->last_video_delay_ms + kMaxVideoDiffMs) { |
| 197 | // Don't increase the delay too much at once |
| 198 | video_delay_ms = |
| 199 | channel_delay_->last_video_delay_ms + kMaxVideoDiffMs; |
| 200 | } |
| 201 | // Verify we don't go above the maximum allowed delay |
| 202 | if (video_delay_ms > kMaxDelay) { |
| 203 | video_delay_ms = kMaxDelay; |
| 204 | } |
| 205 | } else { |
| 206 | if (video_delay_ms < |
| 207 | channel_delay_->last_video_delay_ms - kMaxVideoDiffMs) { |
| 208 | // Don't decrease the delay too much at once |
| 209 | video_delay_ms = |
| 210 | channel_delay_->last_video_delay_ms - kMaxVideoDiffMs; |
| 211 | } |
| 212 | // Verify we don't go below the minimum delay |
| 213 | if (video_delay_ms < *total_video_delay_target_ms) { |
| 214 | video_delay_ms = *total_video_delay_target_ms; |
| 215 | } |
| 216 | } |
| 217 | // Store the values |
| 218 | channel_delay_->extra_video_delay_ms = |
| 219 | video_delay_ms - *total_video_delay_target_ms; |
| 220 | channel_delay_->last_video_delay_ms = video_delay_ms; |
| 221 | channel_delay_->last_sync_delay = -1; |
| 222 | } |
| 223 | } |
| 224 | |
| 225 | WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_, |
| 226 | "Sync video delay %d ms for video channel and audio delay %d for audio " |
| 227 | "channel %d", |
| 228 | video_delay_ms, channel_delay_->extra_audio_delay_ms, audio_channel_id_); |
| 229 | |
| 230 | *extra_audio_delay_ms = channel_delay_->extra_audio_delay_ms; |
| 231 | |
| 232 | if (video_delay_ms < 0) { |
| 233 | video_delay_ms = 0; |
| 234 | } |
| 235 | *total_video_delay_target_ms = |
| 236 | (*total_video_delay_target_ms > video_delay_ms) ? |
| 237 | *total_video_delay_target_ms : video_delay_ms; |
| 238 | return 0; |
| 239 | } |
| 240 | } // namespace webrtc |