andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
pbos@webrtc.org | 281cff8 | 2013-05-17 13:44:48 +0000 | [diff] [blame] | 11 | #include "webrtc/video_engine/vie_sync_module.h" |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 12 | |
wu@webrtc.org | 7fc75bb | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 13 | #include "webrtc/modules/rtp_rtcp/interface/rtp_receiver.h" |
pbos@webrtc.org | 281cff8 | 2013-05-17 13:44:48 +0000 | [diff] [blame] | 14 | #include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h" |
| 15 | #include "webrtc/modules/video_coding/main/interface/video_coding.h" |
| 16 | #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" |
pbos@webrtc.org | 3468f20 | 2014-05-14 08:02:22 +0000 | [diff] [blame] | 17 | #include "webrtc/system_wrappers/interface/logging.h" |
pbos@webrtc.org | 281cff8 | 2013-05-17 13:44:48 +0000 | [diff] [blame] | 18 | #include "webrtc/system_wrappers/interface/trace_event.h" |
| 19 | #include "webrtc/video_engine/stream_synchronization.h" |
| 20 | #include "webrtc/video_engine/vie_channel.h" |
| 21 | #include "webrtc/voice_engine/include/voe_video_sync.h" |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 22 | |
| 23 | namespace webrtc { |
| 24 | |
| 25 | enum { kSyncInterval = 1000}; |
| 26 | |
| 27 | int UpdateMeasurements(StreamSynchronization::Measurements* stream, |
wu@webrtc.org | 7fc75bb | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 28 | const RtpRtcp& rtp_rtcp, const RtpReceiver& receiver) { |
stefan@webrtc.org | 7e97e4c | 2013-11-08 15:18:52 +0000 | [diff] [blame] | 29 | if (!receiver.Timestamp(&stream->latest_timestamp)) |
| 30 | return -1; |
| 31 | if (!receiver.LastReceivedTimeMs(&stream->latest_receive_time_ms)) |
| 32 | return -1; |
wu@webrtc.org | 093fc0b | 2014-04-24 22:10:24 +0000 | [diff] [blame] | 33 | |
| 34 | uint32_t ntp_secs = 0; |
| 35 | uint32_t ntp_frac = 0; |
| 36 | uint32_t rtp_timestamp = 0; |
| 37 | if (0 != rtp_rtcp.RemoteNTP(&ntp_secs, |
| 38 | &ntp_frac, |
wu@webrtc.org | 7fc75bb | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 39 | NULL, |
| 40 | NULL, |
wu@webrtc.org | 093fc0b | 2014-04-24 22:10:24 +0000 | [diff] [blame] | 41 | &rtp_timestamp)) { |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 42 | return -1; |
| 43 | } |
wu@webrtc.org | 093fc0b | 2014-04-24 22:10:24 +0000 | [diff] [blame] | 44 | |
| 45 | bool new_rtcp_sr = false; |
wu@webrtc.org | d2fb259 | 2014-05-07 17:09:44 +0000 | [diff] [blame] | 46 | if (!UpdateRtcpList( |
wu@webrtc.org | 093fc0b | 2014-04-24 22:10:24 +0000 | [diff] [blame] | 47 | ntp_secs, ntp_frac, rtp_timestamp, &stream->rtcp, &new_rtcp_sr)) { |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 48 | return -1; |
| 49 | } |
wu@webrtc.org | 093fc0b | 2014-04-24 22:10:24 +0000 | [diff] [blame] | 50 | |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 51 | return 0; |
| 52 | } |
| 53 | |
| 54 | ViESyncModule::ViESyncModule(VideoCodingModule* vcm, |
| 55 | ViEChannel* vie_channel) |
| 56 | : data_cs_(CriticalSectionWrapper::CreateCriticalSection()), |
| 57 | vcm_(vcm), |
| 58 | vie_channel_(vie_channel), |
wu@webrtc.org | 7fc75bb | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 59 | video_receiver_(NULL), |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 60 | video_rtp_rtcp_(NULL), |
| 61 | voe_channel_id_(-1), |
| 62 | voe_sync_interface_(NULL), |
| 63 | last_sync_time_(TickTime::Now()), |
| 64 | sync_() { |
| 65 | } |
| 66 | |
| 67 | ViESyncModule::~ViESyncModule() { |
| 68 | } |
| 69 | |
| 70 | int ViESyncModule::ConfigureSync(int voe_channel_id, |
| 71 | VoEVideoSync* voe_sync_interface, |
wu@webrtc.org | 7fc75bb | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 72 | RtpRtcp* video_rtcp_module, |
| 73 | RtpReceiver* video_receiver) { |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 74 | CriticalSectionScoped cs(data_cs_.get()); |
| 75 | voe_channel_id_ = voe_channel_id; |
| 76 | voe_sync_interface_ = voe_sync_interface; |
wu@webrtc.org | 7fc75bb | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 77 | video_receiver_ = video_receiver; |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 78 | video_rtp_rtcp_ = video_rtcp_module; |
| 79 | sync_.reset(new StreamSynchronization(voe_channel_id, vie_channel_->Id())); |
| 80 | |
| 81 | if (!voe_sync_interface) { |
| 82 | voe_channel_id_ = -1; |
| 83 | if (voe_channel_id >= 0) { |
| 84 | // Trying to set a voice channel but no interface exist. |
| 85 | return -1; |
| 86 | } |
| 87 | return 0; |
| 88 | } |
| 89 | return 0; |
| 90 | } |
| 91 | |
| 92 | int ViESyncModule::VoiceChannel() { |
| 93 | return voe_channel_id_; |
| 94 | } |
| 95 | |
pbos@webrtc.org | 67879bc | 2013-04-09 13:41:51 +0000 | [diff] [blame] | 96 | int32_t ViESyncModule::TimeUntilNextProcess() { |
| 97 | return static_cast<int32_t>(kSyncInterval - |
| 98 | (TickTime::Now() - last_sync_time_).Milliseconds()); |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 99 | } |
| 100 | |
pbos@webrtc.org | 67879bc | 2013-04-09 13:41:51 +0000 | [diff] [blame] | 101 | int32_t ViESyncModule::Process() { |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 102 | CriticalSectionScoped cs(data_cs_.get()); |
| 103 | last_sync_time_ = TickTime::Now(); |
| 104 | |
hclam@chromium.org | 9540e2a | 2013-06-14 23:30:58 +0000 | [diff] [blame] | 105 | const int current_video_delay_ms = vcm_->Delay(); |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 106 | |
| 107 | if (voe_channel_id_ == -1) { |
| 108 | return 0; |
| 109 | } |
| 110 | assert(video_rtp_rtcp_ && voe_sync_interface_); |
| 111 | assert(sync_.get()); |
| 112 | |
pwestin@webrtc.org | f272497 | 2013-04-11 20:23:35 +0000 | [diff] [blame] | 113 | int audio_jitter_buffer_delay_ms = 0; |
| 114 | int playout_buffer_delay_ms = 0; |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 115 | if (voe_sync_interface_->GetDelayEstimate(voe_channel_id_, |
pwestin@webrtc.org | f272497 | 2013-04-11 20:23:35 +0000 | [diff] [blame] | 116 | &audio_jitter_buffer_delay_ms, |
| 117 | &playout_buffer_delay_ms) != 0) { |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 118 | return 0; |
| 119 | } |
hclam@chromium.org | 9540e2a | 2013-06-14 23:30:58 +0000 | [diff] [blame] | 120 | const int current_audio_delay_ms = audio_jitter_buffer_delay_ms + |
| 121 | playout_buffer_delay_ms; |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 122 | |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 123 | RtpRtcp* voice_rtp_rtcp = NULL; |
wu@webrtc.org | 7fc75bb | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 124 | RtpReceiver* voice_receiver = NULL; |
| 125 | if (0 != voe_sync_interface_->GetRtpRtcp(voe_channel_id_, &voice_rtp_rtcp, |
| 126 | &voice_receiver)) { |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 127 | return 0; |
| 128 | } |
| 129 | assert(voice_rtp_rtcp); |
wu@webrtc.org | 7fc75bb | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 130 | assert(voice_receiver); |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 131 | |
wu@webrtc.org | 7fc75bb | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 132 | if (UpdateMeasurements(&video_measurement_, *video_rtp_rtcp_, |
| 133 | *video_receiver_) != 0) { |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 134 | return 0; |
| 135 | } |
| 136 | |
wu@webrtc.org | 7fc75bb | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 137 | if (UpdateMeasurements(&audio_measurement_, *voice_rtp_rtcp, |
| 138 | *voice_receiver) != 0) { |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 139 | return 0; |
| 140 | } |
| 141 | |
| 142 | int relative_delay_ms; |
| 143 | // Calculate how much later or earlier the audio stream is compared to video. |
| 144 | if (!sync_->ComputeRelativeDelay(audio_measurement_, video_measurement_, |
| 145 | &relative_delay_ms)) { |
| 146 | return 0; |
| 147 | } |
| 148 | |
hclam@chromium.org | 9540e2a | 2013-06-14 23:30:58 +0000 | [diff] [blame] | 149 | TRACE_COUNTER1("webrtc", "SyncCurrentVideoDelay", current_video_delay_ms); |
| 150 | TRACE_COUNTER1("webrtc", "SyncCurrentAudioDelay", current_audio_delay_ms); |
hclam@chromium.org | 74472fe | 2013-04-09 19:54:10 +0000 | [diff] [blame] | 151 | TRACE_COUNTER1("webrtc", "SyncRelativeDelay", relative_delay_ms); |
hclam@chromium.org | 9540e2a | 2013-06-14 23:30:58 +0000 | [diff] [blame] | 152 | int target_audio_delay_ms = 0; |
hclam@chromium.org | b06dd93 | 2013-06-15 06:51:27 +0000 | [diff] [blame] | 153 | int target_video_delay_ms = current_video_delay_ms; |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 154 | // Calculate the necessary extra audio delay and desired total video |
| 155 | // delay to get the streams in sync. |
stefan@webrtc.org | 64ff6c9 | 2012-11-12 18:51:52 +0000 | [diff] [blame] | 156 | if (!sync_->ComputeDelays(relative_delay_ms, |
hclam@chromium.org | 9540e2a | 2013-06-14 23:30:58 +0000 | [diff] [blame] | 157 | current_audio_delay_ms, |
| 158 | &target_audio_delay_ms, |
| 159 | &target_video_delay_ms)) { |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 160 | return 0; |
| 161 | } |
edjee@google.com | dded206 | 2013-04-04 19:43:34 +0000 | [diff] [blame] | 162 | |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 163 | if (voe_sync_interface_->SetMinimumPlayoutDelay( |
hclam@chromium.org | 9540e2a | 2013-06-14 23:30:58 +0000 | [diff] [blame] | 164 | voe_channel_id_, target_audio_delay_ms) == -1) { |
pbos@webrtc.org | 3468f20 | 2014-05-14 08:02:22 +0000 | [diff] [blame] | 165 | LOG(LS_ERROR) << "Error setting voice delay."; |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 166 | } |
hclam@chromium.org | 9540e2a | 2013-06-14 23:30:58 +0000 | [diff] [blame] | 167 | vcm_->SetMinimumPlayoutDelay(target_video_delay_ms); |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 168 | return 0; |
| 169 | } |
| 170 | |
mikhal@webrtc.org | 78e450f | 2013-03-06 23:29:33 +0000 | [diff] [blame] | 171 | int ViESyncModule::SetTargetBufferingDelay(int target_delay_ms) { |
mikhal@webrtc.org | 9d6fcb3 | 2013-02-15 23:22:18 +0000 | [diff] [blame] | 172 | CriticalSectionScoped cs(data_cs_.get()); |
pbos@webrtc.org | 3468f20 | 2014-05-14 08:02:22 +0000 | [diff] [blame] | 173 | if (!voe_sync_interface_) { |
| 174 | LOG(LS_ERROR) << "voe_sync_interface_ NULL, can't set playout delay."; |
mikhal@webrtc.org | 78e450f | 2013-03-06 23:29:33 +0000 | [diff] [blame] | 175 | return -1; |
| 176 | } |
mikhal@webrtc.org | 9d6fcb3 | 2013-02-15 23:22:18 +0000 | [diff] [blame] | 177 | sync_->SetTargetBufferingDelay(target_delay_ms); |
| 178 | // Setting initial playout delay to voice engine (video engine is updated via |
| 179 | // the VCM interface). |
mikhal@webrtc.org | 9d6fcb3 | 2013-02-15 23:22:18 +0000 | [diff] [blame] | 180 | voe_sync_interface_->SetInitialPlayoutDelay(voe_channel_id_, |
| 181 | target_delay_ms); |
mikhal@webrtc.org | 78e450f | 2013-03-06 23:29:33 +0000 | [diff] [blame] | 182 | return 0; |
mikhal@webrtc.org | 9d6fcb3 | 2013-02-15 23:22:18 +0000 | [diff] [blame] | 183 | } |
| 184 | |
andrew@webrtc.org | b015cbe | 2012-10-22 18:19:23 +0000 | [diff] [blame] | 185 | } // namespace webrtc |