blob: f9b885d490acf6e01fc59dba98a3de4d84bbb08d [file] [log] [blame]
stefan@webrtc.org5f284982012-06-28 07:51:16 +00001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Jonas Olssona4d87372019-07-05 19:08:33 +020011#include "video/stream_synchronization.h"
12
kjellander@webrtc.org0fcaf992015-11-26 15:24:52 +010013#include <algorithm>
14
Åsa Persson8fe22fa2019-11-18 14:10:56 +010015#include "system_wrappers/include/clock.h"
Yves Gerey3e707812018-11-28 16:47:49 +010016#include "system_wrappers/include/ntp_time.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020017#include "test/gtest.h"
stefan@webrtc.org5f284982012-06-28 07:51:16 +000018
19namespace webrtc {
Åsa Persson8fe22fa2019-11-18 14:10:56 +010020namespace {
21constexpr int kMaxAudioDiffMs = 80; // From stream_synchronization.cc
22constexpr int kDefaultAudioFrequency = 8000;
23constexpr int kDefaultVideoFrequency = 90000;
24constexpr int kSmoothingFilter = 4 * 2;
25} // namespace
stefan@webrtc.org5f284982012-06-28 07:51:16 +000026
27class StreamSynchronizationTest : public ::testing::Test {
Åsa Persson8fe22fa2019-11-18 14:10:56 +010028 public:
29 StreamSynchronizationTest()
30 : sync_(0, 0), clock_sender_(98765000), clock_receiver_(43210000) {}
31
stefan@webrtc.org5f284982012-06-28 07:51:16 +000032 protected:
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +000033 // Generates the necessary RTCP measurements and RTP timestamps and computes
34 // the audio and video delays needed to get the two streams in sync.
35 // |audio_delay_ms| and |video_delay_ms| are the number of milliseconds after
36 // capture which the frames are rendered.
37 // |current_audio_delay_ms| is the number of milliseconds which audio is
38 // currently being delayed by the receiver.
39 bool DelayedStreams(int audio_delay_ms,
40 int video_delay_ms,
41 int current_audio_delay_ms,
42 int* extra_audio_delay_ms,
43 int* total_video_delay_ms) {
Yves Gerey665174f2018-06-19 15:03:05 +020044 int audio_frequency =
45 static_cast<int>(kDefaultAudioFrequency * audio_clock_drift_ + 0.5);
Yves Gerey665174f2018-06-19 15:03:05 +020046 int video_frequency =
47 static_cast<int>(kDefaultVideoFrequency * video_clock_drift_ + 0.5);
Åsa Persson8fe22fa2019-11-18 14:10:56 +010048
49 // Generate NTP/RTP timestamp pair for both streams corresponding to RTCP.
asaperssonde9e5ff2016-11-02 07:14:03 -070050 bool new_sr;
stefan@webrtc.org5f284982012-06-28 07:51:16 +000051 StreamSynchronization::Measurements audio;
52 StreamSynchronization::Measurements video;
Åsa Persson8fe22fa2019-11-18 14:10:56 +010053 NtpTime ntp_time = clock_sender_.CurrentNtpTime();
asaperssonfe50b4d2016-12-22 07:53:51 -080054 uint32_t rtp_timestamp =
Åsa Persson8fe22fa2019-11-18 14:10:56 +010055 clock_sender_.CurrentTime().ms() * audio_frequency / 1000;
asaperssonfe50b4d2016-12-22 07:53:51 -080056 EXPECT_TRUE(audio.rtp_to_ntp.UpdateMeasurements(
57 ntp_time.seconds(), ntp_time.fractions(), rtp_timestamp, &new_sr));
Åsa Persson8fe22fa2019-11-18 14:10:56 +010058 clock_sender_.AdvanceTimeMilliseconds(100);
59 clock_receiver_.AdvanceTimeMilliseconds(100);
60 ntp_time = clock_sender_.CurrentNtpTime();
61 rtp_timestamp = clock_sender_.CurrentTime().ms() * video_frequency / 1000;
asaperssonfe50b4d2016-12-22 07:53:51 -080062 EXPECT_TRUE(video.rtp_to_ntp.UpdateMeasurements(
63 ntp_time.seconds(), ntp_time.fractions(), rtp_timestamp, &new_sr));
Åsa Persson8fe22fa2019-11-18 14:10:56 +010064 clock_sender_.AdvanceTimeMilliseconds(900);
65 clock_receiver_.AdvanceTimeMilliseconds(900);
66 ntp_time = clock_sender_.CurrentNtpTime();
67 rtp_timestamp = clock_sender_.CurrentTime().ms() * audio_frequency / 1000;
asaperssonfe50b4d2016-12-22 07:53:51 -080068 EXPECT_TRUE(audio.rtp_to_ntp.UpdateMeasurements(
69 ntp_time.seconds(), ntp_time.fractions(), rtp_timestamp, &new_sr));
Åsa Persson8fe22fa2019-11-18 14:10:56 +010070 clock_sender_.AdvanceTimeMilliseconds(100);
71 clock_receiver_.AdvanceTimeMilliseconds(100);
72 ntp_time = clock_sender_.CurrentNtpTime();
73 rtp_timestamp = clock_sender_.CurrentTime().ms() * video_frequency / 1000;
asaperssonfe50b4d2016-12-22 07:53:51 -080074 EXPECT_TRUE(video.rtp_to_ntp.UpdateMeasurements(
75 ntp_time.seconds(), ntp_time.fractions(), rtp_timestamp, &new_sr));
Åsa Persson8fe22fa2019-11-18 14:10:56 +010076 clock_sender_.AdvanceTimeMilliseconds(900);
77 clock_receiver_.AdvanceTimeMilliseconds(900);
stefan@webrtc.org5f284982012-06-28 07:51:16 +000078
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +000079 // Capture an audio and a video frame at the same time.
asaperssonb7e7b492016-11-17 02:27:14 -080080 audio.latest_timestamp =
Åsa Persson8fe22fa2019-11-18 14:10:56 +010081 clock_sender_.CurrentTime().ms() * audio_frequency / 1000;
asaperssonb7e7b492016-11-17 02:27:14 -080082 video.latest_timestamp =
Åsa Persson8fe22fa2019-11-18 14:10:56 +010083 clock_sender_.CurrentTime().ms() * video_frequency / 1000;
stefan@webrtc.org5f284982012-06-28 07:51:16 +000084
85 if (audio_delay_ms > video_delay_ms) {
86 // Audio later than video.
Åsa Persson8fe22fa2019-11-18 14:10:56 +010087 clock_receiver_.AdvanceTimeMilliseconds(video_delay_ms);
88 video.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
89 clock_receiver_.AdvanceTimeMilliseconds(audio_delay_ms - video_delay_ms);
90 audio.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
stefan@webrtc.org5f284982012-06-28 07:51:16 +000091 } else {
92 // Video later than audio.
Åsa Persson8fe22fa2019-11-18 14:10:56 +010093 clock_receiver_.AdvanceTimeMilliseconds(audio_delay_ms);
94 audio.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
95 clock_receiver_.AdvanceTimeMilliseconds(video_delay_ms - audio_delay_ms);
96 video.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
stefan@webrtc.org5f284982012-06-28 07:51:16 +000097 }
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +000098 int relative_delay_ms;
99 StreamSynchronization::ComputeRelativeDelay(audio, video,
100 &relative_delay_ms);
101 EXPECT_EQ(video_delay_ms - audio_delay_ms, relative_delay_ms);
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100102 return sync_.ComputeDelays(relative_delay_ms, current_audio_delay_ms,
103 extra_audio_delay_ms, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000104 }
105
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000106 // Simulate audio playback 300 ms after capture and video rendering 100 ms
107 // after capture. Verify that the correct extra delays are calculated for
108 // audio and video, and that they change correctly when we simulate that
109 // NetEQ or the VCM adds more delay to the streams.
110 // TODO(holmer): This is currently wrong! We should simply change
111 // audio_delay_ms or video_delay_ms since those now include VCM and NetEQ
112 // delays.
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000113 void BothDelayedAudioLaterTest(int base_target_delay) {
114 int current_audio_delay_ms = base_target_delay;
115 int audio_delay_ms = base_target_delay + 300;
116 int video_delay_ms = base_target_delay + 100;
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000117 int extra_audio_delay_ms = 0;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000118 int total_video_delay_ms = base_target_delay;
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000119 int filtered_move = (audio_delay_ms - video_delay_ms) / kSmoothingFilter;
pwestin@webrtc.org4e545b32013-04-26 15:23:34 +0000120 const int kNeteqDelayIncrease = 50;
121 const int kNeteqDelayDecrease = 10;
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000122
Yves Gerey665174f2018-06-19 15:03:05 +0200123 EXPECT_TRUE(DelayedStreams(audio_delay_ms, video_delay_ms,
124 current_audio_delay_ms, &extra_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000125 &total_video_delay_ms));
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000126 EXPECT_EQ(base_target_delay + filtered_move, total_video_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000127 EXPECT_EQ(base_target_delay, extra_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000128 current_audio_delay_ms = extra_audio_delay_ms;
129
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100130 clock_sender_.AdvanceTimeMilliseconds(1000);
131 clock_receiver_.AdvanceTimeMilliseconds(
132 1000 - std::max(audio_delay_ms, video_delay_ms));
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000133 // Simulate base_target_delay minimum delay in the VCM.
134 total_video_delay_ms = base_target_delay;
Yves Gerey665174f2018-06-19 15:03:05 +0200135 EXPECT_TRUE(DelayedStreams(audio_delay_ms, video_delay_ms,
136 current_audio_delay_ms, &extra_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000137 &total_video_delay_ms));
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000138 EXPECT_EQ(base_target_delay + 2 * filtered_move, total_video_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000139 EXPECT_EQ(base_target_delay, extra_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000140 current_audio_delay_ms = extra_audio_delay_ms;
141
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100142 clock_sender_.AdvanceTimeMilliseconds(1000);
143 clock_receiver_.AdvanceTimeMilliseconds(
144 1000 - std::max(audio_delay_ms, video_delay_ms));
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000145 // Simulate base_target_delay minimum delay in the VCM.
146 total_video_delay_ms = base_target_delay;
Yves Gerey665174f2018-06-19 15:03:05 +0200147 EXPECT_TRUE(DelayedStreams(audio_delay_ms, video_delay_ms,
148 current_audio_delay_ms, &extra_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000149 &total_video_delay_ms));
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000150 EXPECT_EQ(base_target_delay + 3 * filtered_move, total_video_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000151 EXPECT_EQ(base_target_delay, extra_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000152
153 // Simulate that NetEQ introduces some audio delay.
pwestin@webrtc.org4e545b32013-04-26 15:23:34 +0000154 current_audio_delay_ms = base_target_delay + kNeteqDelayIncrease;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100155 clock_sender_.AdvanceTimeMilliseconds(1000);
156 clock_receiver_.AdvanceTimeMilliseconds(
157 1000 - std::max(audio_delay_ms, video_delay_ms));
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000158 // Simulate base_target_delay minimum delay in the VCM.
159 total_video_delay_ms = base_target_delay;
Yves Gerey665174f2018-06-19 15:03:05 +0200160 EXPECT_TRUE(DelayedStreams(audio_delay_ms, video_delay_ms,
161 current_audio_delay_ms, &extra_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000162 &total_video_delay_ms));
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000163 filtered_move = 3 * filtered_move +
Yves Gerey665174f2018-06-19 15:03:05 +0200164 (kNeteqDelayIncrease + audio_delay_ms - video_delay_ms) /
165 kSmoothingFilter;
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000166 EXPECT_EQ(base_target_delay + filtered_move, total_video_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000167 EXPECT_EQ(base_target_delay, extra_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000168
169 // Simulate that NetEQ reduces its delay.
pwestin@webrtc.org4e545b32013-04-26 15:23:34 +0000170 current_audio_delay_ms = base_target_delay + kNeteqDelayDecrease;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100171 clock_sender_.AdvanceTimeMilliseconds(1000);
172 clock_receiver_.AdvanceTimeMilliseconds(
173 1000 - std::max(audio_delay_ms, video_delay_ms));
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000174 // Simulate base_target_delay minimum delay in the VCM.
175 total_video_delay_ms = base_target_delay;
Yves Gerey665174f2018-06-19 15:03:05 +0200176 EXPECT_TRUE(DelayedStreams(audio_delay_ms, video_delay_ms,
177 current_audio_delay_ms, &extra_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000178 &total_video_delay_ms));
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000179
180 filtered_move = filtered_move +
Yves Gerey665174f2018-06-19 15:03:05 +0200181 (kNeteqDelayDecrease + audio_delay_ms - video_delay_ms) /
182 kSmoothingFilter;
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000183
184 EXPECT_EQ(base_target_delay + filtered_move, total_video_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000185 EXPECT_EQ(base_target_delay, extra_audio_delay_ms);
186 }
187
188 void BothDelayedVideoLaterTest(int base_target_delay) {
189 int current_audio_delay_ms = base_target_delay;
190 int audio_delay_ms = base_target_delay + 100;
191 int video_delay_ms = base_target_delay + 300;
192 int extra_audio_delay_ms = 0;
193 int total_video_delay_ms = base_target_delay;
194
Yves Gerey665174f2018-06-19 15:03:05 +0200195 EXPECT_TRUE(DelayedStreams(audio_delay_ms, video_delay_ms,
196 current_audio_delay_ms, &extra_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000197 &total_video_delay_ms));
198 EXPECT_EQ(base_target_delay, total_video_delay_ms);
199 // The audio delay is not allowed to change more than this in 1 second.
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000200 EXPECT_GE(base_target_delay + kMaxAudioDiffMs, extra_audio_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000201 current_audio_delay_ms = extra_audio_delay_ms;
202 int current_extra_delay_ms = extra_audio_delay_ms;
203
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100204 clock_sender_.AdvanceTimeMilliseconds(1000);
205 clock_receiver_.AdvanceTimeMilliseconds(800);
Yves Gerey665174f2018-06-19 15:03:05 +0200206 EXPECT_TRUE(DelayedStreams(audio_delay_ms, video_delay_ms,
207 current_audio_delay_ms, &extra_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000208 &total_video_delay_ms));
209 EXPECT_EQ(base_target_delay, total_video_delay_ms);
210 // The audio delay is not allowed to change more than the half of the
211 // required change in delay.
Yves Gerey665174f2018-06-19 15:03:05 +0200212 EXPECT_EQ(current_extra_delay_ms +
213 MaxAudioDelayIncrease(
214 current_audio_delay_ms,
215 base_target_delay + video_delay_ms - audio_delay_ms),
216 extra_audio_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000217 current_audio_delay_ms = extra_audio_delay_ms;
218 current_extra_delay_ms = extra_audio_delay_ms;
219
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100220 clock_sender_.AdvanceTimeMilliseconds(1000);
221 clock_receiver_.AdvanceTimeMilliseconds(800);
Yves Gerey665174f2018-06-19 15:03:05 +0200222 EXPECT_TRUE(DelayedStreams(audio_delay_ms, video_delay_ms,
223 current_audio_delay_ms, &extra_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000224 &total_video_delay_ms));
225 EXPECT_EQ(base_target_delay, total_video_delay_ms);
226 // The audio delay is not allowed to change more than the half of the
227 // required change in delay.
Yves Gerey665174f2018-06-19 15:03:05 +0200228 EXPECT_EQ(current_extra_delay_ms +
229 MaxAudioDelayIncrease(
230 current_audio_delay_ms,
231 base_target_delay + video_delay_ms - audio_delay_ms),
232 extra_audio_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000233 current_extra_delay_ms = extra_audio_delay_ms;
234
235 // Simulate that NetEQ for some reason reduced the delay.
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000236 current_audio_delay_ms = base_target_delay + 10;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100237 clock_sender_.AdvanceTimeMilliseconds(1000);
238 clock_receiver_.AdvanceTimeMilliseconds(800);
Yves Gerey665174f2018-06-19 15:03:05 +0200239 EXPECT_TRUE(DelayedStreams(audio_delay_ms, video_delay_ms,
240 current_audio_delay_ms, &extra_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000241 &total_video_delay_ms));
242 EXPECT_EQ(base_target_delay, total_video_delay_ms);
243 // Since we only can ask NetEQ for a certain amount of extra delay, and
244 // we only measure the total NetEQ delay, we will ask for additional delay
245 // here to try to stay in sync.
Yves Gerey665174f2018-06-19 15:03:05 +0200246 EXPECT_EQ(current_extra_delay_ms +
247 MaxAudioDelayIncrease(
248 current_audio_delay_ms,
249 base_target_delay + video_delay_ms - audio_delay_ms),
250 extra_audio_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000251 current_extra_delay_ms = extra_audio_delay_ms;
252
253 // Simulate that NetEQ for some reason significantly increased the delay.
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000254 current_audio_delay_ms = base_target_delay + 350;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100255 clock_sender_.AdvanceTimeMilliseconds(1000);
256 clock_receiver_.AdvanceTimeMilliseconds(800);
Yves Gerey665174f2018-06-19 15:03:05 +0200257 EXPECT_TRUE(DelayedStreams(audio_delay_ms, video_delay_ms,
258 current_audio_delay_ms, &extra_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000259 &total_video_delay_ms));
260 EXPECT_EQ(base_target_delay, total_video_delay_ms);
261 // The audio delay is not allowed to change more than the half of the
262 // required change in delay.
Yves Gerey665174f2018-06-19 15:03:05 +0200263 EXPECT_EQ(current_extra_delay_ms +
264 MaxAudioDelayIncrease(
265 current_audio_delay_ms,
266 base_target_delay + video_delay_ms - audio_delay_ms),
267 extra_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000268 }
269
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000270 int MaxAudioDelayIncrease(int current_audio_delay_ms, int delay_ms) {
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000271 return std::min((delay_ms - current_audio_delay_ms) / kSmoothingFilter,
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100272 kMaxAudioDiffMs);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000273 }
274
275 int MaxAudioDelayDecrease(int current_audio_delay_ms, int delay_ms) {
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000276 return std::max((delay_ms - current_audio_delay_ms) / kSmoothingFilter,
277 -kMaxAudioDiffMs);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000278 }
279
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100280 StreamSynchronization sync_;
281 SimulatedClock clock_sender_;
282 SimulatedClock clock_receiver_;
283 double audio_clock_drift_ = 1.0;
284 double video_clock_drift_ = 1.0;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000285};
286
287TEST_F(StreamSynchronizationTest, NoDelay) {
288 uint32_t current_audio_delay_ms = 0;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000289 int extra_audio_delay_ms = 0;
290 int total_video_delay_ms = 0;
291
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000292 EXPECT_FALSE(DelayedStreams(0, 0, current_audio_delay_ms,
293 &extra_audio_delay_ms, &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000294 EXPECT_EQ(0, extra_audio_delay_ms);
295 EXPECT_EQ(0, total_video_delay_ms);
296}
297
298TEST_F(StreamSynchronizationTest, VideoDelay) {
299 uint32_t current_audio_delay_ms = 0;
300 int delay_ms = 200;
301 int extra_audio_delay_ms = 0;
302 int total_video_delay_ms = 0;
303
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000304 EXPECT_TRUE(DelayedStreams(delay_ms, 0, current_audio_delay_ms,
305 &extra_audio_delay_ms, &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000306 EXPECT_EQ(0, extra_audio_delay_ms);
307 // The video delay is not allowed to change more than this in 1 second.
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000308 EXPECT_EQ(delay_ms / kSmoothingFilter, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000309
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100310 clock_sender_.AdvanceTimeMilliseconds(1000);
311 clock_receiver_.AdvanceTimeMilliseconds(800);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000312 // Simulate 0 minimum delay in the VCM.
313 total_video_delay_ms = 0;
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000314 EXPECT_TRUE(DelayedStreams(delay_ms, 0, current_audio_delay_ms,
315 &extra_audio_delay_ms, &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000316 EXPECT_EQ(0, extra_audio_delay_ms);
317 // The video delay is not allowed to change more than this in 1 second.
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000318 EXPECT_EQ(2 * delay_ms / kSmoothingFilter, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000319
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100320 clock_sender_.AdvanceTimeMilliseconds(1000);
321 clock_receiver_.AdvanceTimeMilliseconds(800);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000322 // Simulate 0 minimum delay in the VCM.
323 total_video_delay_ms = 0;
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000324 EXPECT_TRUE(DelayedStreams(delay_ms, 0, current_audio_delay_ms,
325 &extra_audio_delay_ms, &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000326 EXPECT_EQ(0, extra_audio_delay_ms);
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000327 EXPECT_EQ(3 * delay_ms / kSmoothingFilter, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000328}
329
330TEST_F(StreamSynchronizationTest, AudioDelay) {
331 int current_audio_delay_ms = 0;
332 int delay_ms = 200;
333 int extra_audio_delay_ms = 0;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000334 int total_video_delay_ms = 0;
335
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000336 EXPECT_TRUE(DelayedStreams(0, delay_ms, current_audio_delay_ms,
337 &extra_audio_delay_ms, &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000338 EXPECT_EQ(0, total_video_delay_ms);
339 // The audio delay is not allowed to change more than this in 1 second.
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000340 EXPECT_EQ(delay_ms / kSmoothingFilter, extra_audio_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000341 current_audio_delay_ms = extra_audio_delay_ms;
andrew@webrtc.orgd7a71d02012-08-01 01:40:02 +0000342 int current_extra_delay_ms = extra_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000343
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100344 clock_sender_.AdvanceTimeMilliseconds(1000);
345 clock_receiver_.AdvanceTimeMilliseconds(800);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000346 EXPECT_TRUE(DelayedStreams(0, delay_ms, current_audio_delay_ms,
347 &extra_audio_delay_ms, &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000348 EXPECT_EQ(0, total_video_delay_ms);
349 // The audio delay is not allowed to change more than the half of the required
350 // change in delay.
351 EXPECT_EQ(current_extra_delay_ms +
Yves Gerey665174f2018-06-19 15:03:05 +0200352 MaxAudioDelayIncrease(current_audio_delay_ms, delay_ms),
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000353 extra_audio_delay_ms);
354 current_audio_delay_ms = extra_audio_delay_ms;
355 current_extra_delay_ms = extra_audio_delay_ms;
356
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100357 clock_sender_.AdvanceTimeMilliseconds(1000);
358 clock_receiver_.AdvanceTimeMilliseconds(800);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000359 EXPECT_TRUE(DelayedStreams(0, delay_ms, current_audio_delay_ms,
360 &extra_audio_delay_ms, &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000361 EXPECT_EQ(0, total_video_delay_ms);
362 // The audio delay is not allowed to change more than the half of the required
363 // change in delay.
364 EXPECT_EQ(current_extra_delay_ms +
Yves Gerey665174f2018-06-19 15:03:05 +0200365 MaxAudioDelayIncrease(current_audio_delay_ms, delay_ms),
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000366 extra_audio_delay_ms);
367 current_extra_delay_ms = extra_audio_delay_ms;
368
369 // Simulate that NetEQ for some reason reduced the delay.
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000370 current_audio_delay_ms = 10;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100371 clock_sender_.AdvanceTimeMilliseconds(1000);
372 clock_receiver_.AdvanceTimeMilliseconds(800);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000373 EXPECT_TRUE(DelayedStreams(0, delay_ms, current_audio_delay_ms,
374 &extra_audio_delay_ms, &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000375 EXPECT_EQ(0, total_video_delay_ms);
376 // Since we only can ask NetEQ for a certain amount of extra delay, and
377 // we only measure the total NetEQ delay, we will ask for additional delay
378 // here to try to
379 EXPECT_EQ(current_extra_delay_ms +
Yves Gerey665174f2018-06-19 15:03:05 +0200380 MaxAudioDelayIncrease(current_audio_delay_ms, delay_ms),
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000381 extra_audio_delay_ms);
382 current_extra_delay_ms = extra_audio_delay_ms;
383
384 // Simulate that NetEQ for some reason significantly increased the delay.
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000385 current_audio_delay_ms = 350;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100386 clock_sender_.AdvanceTimeMilliseconds(1000);
387 clock_receiver_.AdvanceTimeMilliseconds(800);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000388 EXPECT_TRUE(DelayedStreams(0, delay_ms, current_audio_delay_ms,
389 &extra_audio_delay_ms, &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000390 EXPECT_EQ(0, total_video_delay_ms);
391 // The audio delay is not allowed to change more than the half of the required
392 // change in delay.
393 EXPECT_EQ(current_extra_delay_ms +
Yves Gerey665174f2018-06-19 15:03:05 +0200394 MaxAudioDelayDecrease(current_audio_delay_ms, delay_ms),
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000395 extra_audio_delay_ms);
396}
397
398TEST_F(StreamSynchronizationTest, BothDelayedVideoLater) {
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000399 BothDelayedVideoLaterTest(0);
400}
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000401
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000402TEST_F(StreamSynchronizationTest, BothDelayedVideoLaterAudioClockDrift) {
403 audio_clock_drift_ = 1.05;
404 BothDelayedVideoLaterTest(0);
405}
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000406
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000407TEST_F(StreamSynchronizationTest, BothDelayedVideoLaterVideoClockDrift) {
408 video_clock_drift_ = 1.05;
409 BothDelayedVideoLaterTest(0);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000410}
411
412TEST_F(StreamSynchronizationTest, BothDelayedAudioLater) {
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000413 BothDelayedAudioLaterTest(0);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000414}
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000415
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000416TEST_F(StreamSynchronizationTest, BothDelayedAudioClockDrift) {
417 audio_clock_drift_ = 1.05;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000418 BothDelayedAudioLaterTest(0);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000419}
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000420
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000421TEST_F(StreamSynchronizationTest, BothDelayedVideoClockDrift) {
422 video_clock_drift_ = 1.05;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000423 BothDelayedAudioLaterTest(0);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000424}
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000425
426TEST_F(StreamSynchronizationTest, BaseDelay) {
427 int base_target_delay_ms = 2000;
428 int current_audio_delay_ms = 2000;
429 int extra_audio_delay_ms = 0;
430 int total_video_delay_ms = base_target_delay_ms;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100431 sync_.SetTargetBufferingDelay(base_target_delay_ms);
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000432 // We are in sync don't change.
433 EXPECT_FALSE(DelayedStreams(base_target_delay_ms, base_target_delay_ms,
Yves Gerey665174f2018-06-19 15:03:05 +0200434 current_audio_delay_ms, &extra_audio_delay_ms,
435 &total_video_delay_ms));
mikhal@webrtc.org0d8d0102013-02-22 19:30:44 +0000436 // Triggering another call with the same values. Delay should not be modified.
437 base_target_delay_ms = 2000;
438 current_audio_delay_ms = base_target_delay_ms;
439 total_video_delay_ms = base_target_delay_ms;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100440 sync_.SetTargetBufferingDelay(base_target_delay_ms);
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000441 // We are in sync don't change.
442 EXPECT_FALSE(DelayedStreams(base_target_delay_ms, base_target_delay_ms,
Yves Gerey665174f2018-06-19 15:03:05 +0200443 current_audio_delay_ms, &extra_audio_delay_ms,
444 &total_video_delay_ms));
mikhal@webrtc.org0d8d0102013-02-22 19:30:44 +0000445 // Changing delay value - intended to test this module only. In practice it
446 // would take VoE time to adapt.
447 base_target_delay_ms = 5000;
448 current_audio_delay_ms = base_target_delay_ms;
449 total_video_delay_ms = base_target_delay_ms;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100450 sync_.SetTargetBufferingDelay(base_target_delay_ms);
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000451 // We are in sync don't change.
452 EXPECT_FALSE(DelayedStreams(base_target_delay_ms, base_target_delay_ms,
Yves Gerey665174f2018-06-19 15:03:05 +0200453 current_audio_delay_ms, &extra_audio_delay_ms,
454 &total_video_delay_ms));
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000455}
456
457TEST_F(StreamSynchronizationTest, BothDelayedAudioLaterWithBaseDelay) {
458 int base_target_delay_ms = 3000;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100459 sync_.SetTargetBufferingDelay(base_target_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000460 BothDelayedAudioLaterTest(base_target_delay_ms);
461}
462
463TEST_F(StreamSynchronizationTest, BothDelayedAudioClockDriftWithBaseDelay) {
464 int base_target_delay_ms = 3000;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100465 sync_.SetTargetBufferingDelay(base_target_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000466 audio_clock_drift_ = 1.05;
467 BothDelayedAudioLaterTest(base_target_delay_ms);
468}
469
470TEST_F(StreamSynchronizationTest, BothDelayedVideoClockDriftWithBaseDelay) {
471 int base_target_delay_ms = 3000;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100472 sync_.SetTargetBufferingDelay(base_target_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000473 video_clock_drift_ = 1.05;
474 BothDelayedAudioLaterTest(base_target_delay_ms);
475}
476
477TEST_F(StreamSynchronizationTest, BothDelayedVideoLaterWithBaseDelay) {
478 int base_target_delay_ms = 2000;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100479 sync_.SetTargetBufferingDelay(base_target_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000480 BothDelayedVideoLaterTest(base_target_delay_ms);
481}
482
483TEST_F(StreamSynchronizationTest,
484 BothDelayedVideoLaterAudioClockDriftWithBaseDelay) {
485 int base_target_delay_ms = 2000;
486 audio_clock_drift_ = 1.05;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100487 sync_.SetTargetBufferingDelay(base_target_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000488 BothDelayedVideoLaterTest(base_target_delay_ms);
489}
490
491TEST_F(StreamSynchronizationTest,
492 BothDelayedVideoLaterVideoClockDriftWithBaseDelay) {
493 int base_target_delay_ms = 2000;
494 video_clock_drift_ = 1.05;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100495 sync_.SetTargetBufferingDelay(base_target_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000496 BothDelayedVideoLaterTest(base_target_delay_ms);
497}
498
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000499} // namespace webrtc