First step in refactoring audio/video synchronization. Adds unittests.
BUG=
TEST=stream_synchronization_unittest
Review URL: https://webrtc-codereview.appspot.com/669005
git-svn-id: http://webrtc.googlecode.com/svn/trunk@2455 4adac7df-926f-26a2-2b94-8c16560cd09d
diff --git a/src/video_engine/stream_synchronization.cc b/src/video_engine/stream_synchronization.cc
new file mode 100644
index 0000000..1ba1f09
--- /dev/null
+++ b/src/video_engine/stream_synchronization.cc
@@ -0,0 +1,240 @@
+/*
+ * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video_engine/stream_synchronization.h"
+#include "system_wrappers/interface/trace.h"
+
+namespace webrtc {
+
+enum { kMaxVideoDiffMs = 80 };
+enum { kMaxAudioDiffMs = 80 };
+enum { kMaxDelay = 1500 };
+
+const float FracMS = 4.294967296E6f;
+
+struct ViESyncDelay {
+ ViESyncDelay() {
+ extra_video_delay_ms = 0;
+ last_video_delay_ms = 0;
+ extra_audio_delay_ms = 0;
+ last_sync_delay = 0;
+ network_delay = 120;
+ }
+
+ int extra_video_delay_ms;
+ int last_video_delay_ms;
+ int extra_audio_delay_ms;
+ int last_sync_delay;
+ int network_delay;
+};
+
+StreamSynchronization::StreamSynchronization(int audio_channel_id,
+ int video_channel_id)
+ : channel_delay_(new ViESyncDelay),
+ audio_channel_id_(audio_channel_id),
+ video_channel_id_(video_channel_id) {}
+
+StreamSynchronization::~StreamSynchronization() {
+ delete channel_delay_;
+}
+
+int StreamSynchronization::ComputeDelays(const Measurements& audio,
+ int current_audio_delay_ms,
+ int* extra_audio_delay_ms,
+ const Measurements& video,
+ int* total_video_delay_target_ms) {
+ // ReceivedNTPxxx is NTP at sender side when sent.
+ // RTCPArrivalTimexxx is NTP at receiver side when received.
+ // can't use ConvertNTPTimeToMS since calculation can be
+ // negative
+ int NTPdiff = (audio.received_ntp_secs - video.received_ntp_secs)
+ * 1000; // ms
+ float ntp_diff_frac = audio.received_ntp_frac / FracMS -
+ video.received_ntp_frac / FracMS;
+ if (ntp_diff_frac > 0.0f)
+ NTPdiff += static_cast<int>(ntp_diff_frac + 0.5f);
+ else
+ NTPdiff += static_cast<int>(ntp_diff_frac - 0.5f);
+
+ int RTCPdiff = (audio.rtcp_arrivaltime_secs - video.rtcp_arrivaltime_secs)
+ * 1000; // ms
+ float rtcp_diff_frac = audio.rtcp_arrivaltime_frac / FracMS -
+ video.rtcp_arrivaltime_frac / FracMS;
+ if (rtcp_diff_frac > 0.0f)
+ RTCPdiff += static_cast<int>(rtcp_diff_frac + 0.5f);
+ else
+ RTCPdiff += static_cast<int>(rtcp_diff_frac - 0.5f);
+
+ int diff = NTPdiff - RTCPdiff;
+ // if diff is + video is behind
+ if (diff < -1000 || diff > 1000) {
+ // unresonable ignore value.
+ return -1;
+ }
+ channel_delay_->network_delay = diff;
+
+ WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_,
+ "Audio delay is: %d for voice channel: %d",
+ current_audio_delay_ms, audio_channel_id_);
+ WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_,
+ "Network delay diff is: %d for voice channel: %d",
+ channel_delay_->network_delay, audio_channel_id_);
+ // Calculate the difference between the lowest possible video delay and
+ // the current audio delay.
+ int current_diff_ms = *total_video_delay_target_ms - current_audio_delay_ms +
+ channel_delay_->network_delay;
+ WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_,
+ "Current diff is: %d for audio channel: %d",
+ current_diff_ms, audio_channel_id_);
+
+ int video_delay_ms = 0;
+ if (current_diff_ms > 0) {
+ // The minimum video delay is longer than the current audio delay.
+ // We need to decrease extra video delay, if we have added extra delay
+ // earlier, or add extra audio delay.
+ if (channel_delay_->extra_video_delay_ms > 0) {
+ // We have extra delay added to ViE. Reduce this delay before adding
+ // extra delay to VoE.
+
+ // This is the desired delay, we can't reduce more than this.
+ video_delay_ms = *total_video_delay_target_ms;
+
+ // Check that we don't reduce the delay more than what is allowed.
+ if (video_delay_ms <
+ channel_delay_->last_video_delay_ms - kMaxVideoDiffMs) {
+ video_delay_ms =
+ channel_delay_->last_video_delay_ms - kMaxVideoDiffMs;
+ channel_delay_->extra_video_delay_ms =
+ video_delay_ms - *total_video_delay_target_ms;
+ } else {
+ channel_delay_->extra_video_delay_ms = 0;
+ }
+ channel_delay_->last_video_delay_ms = video_delay_ms;
+ channel_delay_->last_sync_delay = -1;
+ channel_delay_->extra_audio_delay_ms = 0;
+ } else { // channel_delay_->extra_video_delay_ms > 0
+ // We have no extra video delay to remove, increase the audio delay.
+ if (channel_delay_->last_sync_delay >= 0) {
+ // We have increased the audio delay earlier, increase it even more.
+ int audio_diff_ms = current_diff_ms / 2;
+ if (audio_diff_ms > kMaxAudioDiffMs) {
+ // We only allow a maximum change of KMaxAudioDiffMS for audio
+ // due to NetEQ maximum changes.
+ audio_diff_ms = kMaxAudioDiffMs;
+ }
+ // Increase the audio delay
+ channel_delay_->extra_audio_delay_ms += audio_diff_ms;
+
+ // Don't set a too high delay.
+ if (channel_delay_->extra_audio_delay_ms > kMaxDelay) {
+ channel_delay_->extra_audio_delay_ms = kMaxDelay;
+ }
+
+ // Don't add any extra video delay.
+ video_delay_ms = *total_video_delay_target_ms;
+ channel_delay_->extra_video_delay_ms = 0;
+ channel_delay_->last_video_delay_ms = video_delay_ms;
+ channel_delay_->last_sync_delay = 1;
+ } else { // channel_delay_->last_sync_delay >= 0
+ // First time after a delay change, don't add any extra delay.
+ // This is to not toggle back and forth too much.
+ channel_delay_->extra_audio_delay_ms = 0;
+ // Set minimum video delay
+ video_delay_ms = *total_video_delay_target_ms;
+ channel_delay_->extra_video_delay_ms = 0;
+ channel_delay_->last_video_delay_ms = video_delay_ms;
+ channel_delay_->last_sync_delay = 0;
+ }
+ }
+ } else { // if (current_diffMS > 0)
+ // The minimum video delay is lower than the current audio delay.
+ // We need to decrease possible extra audio delay, or
+ // add extra video delay.
+
+ if (channel_delay_->extra_audio_delay_ms > 0) {
+ // We have extra delay in VoiceEngine
+ // Start with decreasing the voice delay
+ int audio_diff_ms = current_diff_ms / 2;
+ if (audio_diff_ms < -1 * kMaxAudioDiffMs) {
+ // Don't change the delay too much at once.
+ audio_diff_ms = -1 * kMaxAudioDiffMs;
+ }
+ // Add the negative difference.
+ channel_delay_->extra_audio_delay_ms += audio_diff_ms;
+
+ if (channel_delay_->extra_audio_delay_ms < 0) {
+ // Negative values not allowed.
+ channel_delay_->extra_audio_delay_ms = 0;
+ channel_delay_->last_sync_delay = 0;
+ } else {
+ // There is more audio delay to use for the next round.
+ channel_delay_->last_sync_delay = 1;
+ }
+
+ // Keep the video delay at the minimum values.
+ video_delay_ms = *total_video_delay_target_ms;
+ channel_delay_->extra_video_delay_ms = 0;
+ channel_delay_->last_video_delay_ms = video_delay_ms;
+ } else { // channel_delay_->extra_audio_delay_ms > 0
+ // We have no extra delay in VoiceEngine, increase the video delay.
+ channel_delay_->extra_audio_delay_ms = 0;
+
+ // Make the difference positive.
+ int video_diff_ms = -1 * current_diff_ms;
+
+ // This is the desired delay.
+ video_delay_ms = *total_video_delay_target_ms + video_diff_ms;
+ if (video_delay_ms > channel_delay_->last_video_delay_ms) {
+ if (video_delay_ms >
+ channel_delay_->last_video_delay_ms + kMaxVideoDiffMs) {
+ // Don't increase the delay too much at once
+ video_delay_ms =
+ channel_delay_->last_video_delay_ms + kMaxVideoDiffMs;
+ }
+ // Verify we don't go above the maximum allowed delay
+ if (video_delay_ms > kMaxDelay) {
+ video_delay_ms = kMaxDelay;
+ }
+ } else {
+ if (video_delay_ms <
+ channel_delay_->last_video_delay_ms - kMaxVideoDiffMs) {
+ // Don't decrease the delay too much at once
+ video_delay_ms =
+ channel_delay_->last_video_delay_ms - kMaxVideoDiffMs;
+ }
+ // Verify we don't go below the minimum delay
+ if (video_delay_ms < *total_video_delay_target_ms) {
+ video_delay_ms = *total_video_delay_target_ms;
+ }
+ }
+ // Store the values
+ channel_delay_->extra_video_delay_ms =
+ video_delay_ms - *total_video_delay_target_ms;
+ channel_delay_->last_video_delay_ms = video_delay_ms;
+ channel_delay_->last_sync_delay = -1;
+ }
+ }
+
+ WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_,
+ "Sync video delay %d ms for video channel and audio delay %d for audio "
+ "channel %d",
+ video_delay_ms, channel_delay_->extra_audio_delay_ms, audio_channel_id_);
+
+ *extra_audio_delay_ms = channel_delay_->extra_audio_delay_ms;
+
+ if (video_delay_ms < 0) {
+ video_delay_ms = 0;
+ }
+ *total_video_delay_target_ms =
+ (*total_video_delay_target_ms > video_delay_ms) ?
+ *total_video_delay_target_ms : video_delay_ms;
+ return 0;
+}
+} // namespace webrtc
diff --git a/src/video_engine/stream_synchronization.h b/src/video_engine/stream_synchronization.h
new file mode 100644
index 0000000..5a88383
--- /dev/null
+++ b/src/video_engine/stream_synchronization.h
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_VIDEO_ENGINE_STREAM_SYNCHRONIZATION_H_
+#define WEBRTC_VIDEO_ENGINE_STREAM_SYNCHRONIZATION_H_
+
+#include "typedefs.h"
+
+namespace webrtc {
+
+struct ViESyncDelay;
+
+class StreamSynchronization {
+ public:
+ struct Measurements {
+ Measurements()
+ : received_ntp_secs(0),
+ received_ntp_frac(0),
+ rtcp_arrivaltime_secs(0),
+ rtcp_arrivaltime_frac(0) {}
+ uint32_t received_ntp_secs;
+ uint32_t received_ntp_frac;
+ uint32_t rtcp_arrivaltime_secs;
+ uint32_t rtcp_arrivaltime_frac;
+ };
+
+ StreamSynchronization(int audio_channel_id, int video_channel_id);
+ ~StreamSynchronization();
+
+ int ComputeDelays(const Measurements& audio,
+ int current_audio_delay_ms,
+ int* extra_audio_delay_ms,
+ const Measurements& video,
+ int* total_video_delay_target_ms);
+
+ private:
+ ViESyncDelay* channel_delay_;
+ int audio_channel_id_;
+ int video_channel_id_;
+};
+
+} // namespace webrtc
+
+#endif // WEBRTC_VIDEO_ENGINE_STREAM_SYNCHRONIZATION_H_
diff --git a/src/video_engine/stream_synchronization_unittest.cc b/src/video_engine/stream_synchronization_unittest.cc
new file mode 100644
index 0000000..d4b002c
--- /dev/null
+++ b/src/video_engine/stream_synchronization_unittest.cc
@@ -0,0 +1,436 @@
+/*
+ * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <math.h>
+#include <algorithm>
+
+#include "gtest/gtest.h"
+#include "video_engine/stream_synchronization.h"
+
+namespace webrtc {
+
+// These correspond to the same constants defined in vie_sync_module.cc.
+enum { kMaxVideoDiffMs = 80 };
+enum { kMaxAudioDiffMs = 80 };
+enum { kMaxDelay = 1500 };
+
+class Time {
+ public:
+ explicit Time(int64_t offset)
+ : kNtpJan1970(2208988800UL),
+ time_now_ms_(offset) {}
+
+ void NowNtp(uint32_t* ntp_secs, uint32_t* ntp_frac) const {
+ *ntp_secs = time_now_ms_ / 1000 + kNtpJan1970;
+ int64_t remainder = time_now_ms_ % 1000;
+ *ntp_frac = static_cast<uint32_t>(
+ static_cast<double>(remainder) / 1000.0 * pow(2.0, 32.0) + 0.5);
+ }
+
+ void IncreaseTimeMs(int64_t inc) {
+ time_now_ms_ += inc;
+ }
+
+ int64_t time_now_ms() const {
+ return time_now_ms_;
+ }
+ private:
+ // January 1970, in NTP seconds.
+ const uint32_t kNtpJan1970;
+ int64_t time_now_ms_;
+};
+
+class StreamSynchronizationTest : public ::testing::Test {
+ protected:
+ virtual void SetUp() {
+ sync_ = new StreamSynchronization(0, 0);
+ send_time_ = new Time(kSendTimeOffsetMs);
+ receive_time_ = new Time(kReceiveTimeOffsetMs);
+ }
+
+ virtual void TearDown() {
+ delete sync_;
+ delete send_time_;
+ delete receive_time_;
+ }
+
+ int DelayedAudio(int delay_ms,
+ int current_audio_delay_ms,
+ int* extra_audio_delay_ms,
+ int* total_video_delay_ms) {
+ StreamSynchronization::Measurements audio;
+ StreamSynchronization::Measurements video;
+ send_time_->NowNtp(&audio.received_ntp_secs, &audio.received_ntp_frac);
+ send_time_->NowNtp(&video.received_ntp_secs, &video.received_ntp_frac);
+ receive_time_->NowNtp(&video.rtcp_arrivaltime_secs,
+ &video.rtcp_arrivaltime_frac);
+ // Audio later than video.
+ receive_time_->IncreaseTimeMs(delay_ms);
+ receive_time_->NowNtp(&audio.rtcp_arrivaltime_secs,
+ &audio.rtcp_arrivaltime_frac);
+ return sync_->ComputeDelays(audio,
+ current_audio_delay_ms,
+ extra_audio_delay_ms,
+ video,
+ total_video_delay_ms);
+ }
+
+ int DelayedVideo(int delay_ms,
+ int current_audio_delay_ms,
+ int* extra_audio_delay_ms,
+ int* total_video_delay_ms) {
+ StreamSynchronization::Measurements audio;
+ StreamSynchronization::Measurements video;
+ send_time_->NowNtp(&audio.received_ntp_secs, &audio.received_ntp_frac);
+ send_time_->NowNtp(&video.received_ntp_secs, &video.received_ntp_frac);
+ receive_time_->NowNtp(&audio.rtcp_arrivaltime_secs,
+ &audio.rtcp_arrivaltime_frac);
+ // Video later than audio.
+ receive_time_->IncreaseTimeMs(delay_ms);
+ receive_time_->NowNtp(&video.rtcp_arrivaltime_secs,
+ &video.rtcp_arrivaltime_frac);
+ return sync_->ComputeDelays(audio,
+ current_audio_delay_ms,
+ extra_audio_delay_ms,
+ video,
+ total_video_delay_ms);
+ }
+
+ int DelayedAudioAndVideo(int audio_delay_ms,
+ int video_delay_ms,
+ int current_audio_delay_ms,
+ int* extra_audio_delay_ms,
+ int* total_video_delay_ms) {
+ StreamSynchronization::Measurements audio;
+ StreamSynchronization::Measurements video;
+ send_time_->NowNtp(&audio.received_ntp_secs, &audio.received_ntp_frac);
+ send_time_->NowNtp(&video.received_ntp_secs, &video.received_ntp_frac);
+
+ if (audio_delay_ms > video_delay_ms) {
+ // Audio later than video.
+ receive_time_->IncreaseTimeMs(video_delay_ms);
+ receive_time_->NowNtp(&video.rtcp_arrivaltime_secs,
+ &video.rtcp_arrivaltime_frac);
+ receive_time_->IncreaseTimeMs(audio_delay_ms - video_delay_ms);
+ receive_time_->NowNtp(&audio.rtcp_arrivaltime_secs,
+ &audio.rtcp_arrivaltime_frac);
+ } else {
+ // Video later than audio.
+ receive_time_->IncreaseTimeMs(audio_delay_ms);
+ receive_time_->NowNtp(&audio.rtcp_arrivaltime_secs,
+ &audio.rtcp_arrivaltime_frac);
+ receive_time_->IncreaseTimeMs(video_delay_ms - audio_delay_ms);
+ receive_time_->NowNtp(&video.rtcp_arrivaltime_secs,
+ &video.rtcp_arrivaltime_frac);
+ }
+ return sync_->ComputeDelays(audio,
+ current_audio_delay_ms,
+ extra_audio_delay_ms,
+ video,
+ total_video_delay_ms);
+ }
+
+ int MaxAudioDelayIncrease(int current_audio_delay_ms, int delay_ms) {
+ return std::min((delay_ms - current_audio_delay_ms) / 2,
+ static_cast<int>(kMaxAudioDiffMs));
+ }
+
+ int MaxAudioDelayDecrease(int current_audio_delay_ms, int delay_ms) {
+ return std::max((delay_ms - current_audio_delay_ms) / 2, -kMaxAudioDiffMs);
+ }
+
+ enum { kSendTimeOffsetMs = 0 };
+ enum { kReceiveTimeOffsetMs = 123456 };
+
+ StreamSynchronization* sync_;
+ Time* send_time_;
+ Time* receive_time_;
+};
+
+TEST_F(StreamSynchronizationTest, NoDelay) {
+ uint32_t current_audio_delay_ms = 0;
+ int delay_ms = 0;
+ int extra_audio_delay_ms = 0;
+ int total_video_delay_ms = 0;
+
+ EXPECT_EQ(0, DelayedAudio(delay_ms, current_audio_delay_ms,
+ &extra_audio_delay_ms, &total_video_delay_ms));
+ EXPECT_EQ(0, extra_audio_delay_ms);
+ EXPECT_EQ(0, total_video_delay_ms);
+}
+
+TEST_F(StreamSynchronizationTest, VideoDelay) {
+ uint32_t current_audio_delay_ms = 0;
+ int delay_ms = 200;
+ int extra_audio_delay_ms = 0;
+ int total_video_delay_ms = 0;
+
+ EXPECT_EQ(0, DelayedAudio(delay_ms, current_audio_delay_ms,
+ &extra_audio_delay_ms, &total_video_delay_ms));
+ EXPECT_EQ(0, extra_audio_delay_ms);
+ // The video delay is not allowed to change more than this in 1 second.
+ EXPECT_EQ(kMaxVideoDiffMs, total_video_delay_ms);
+
+ send_time_->IncreaseTimeMs(1000);
+ receive_time_->IncreaseTimeMs(800);
+ // Simulate 0 minimum delay in the VCM.
+ total_video_delay_ms = 0;
+ EXPECT_EQ(0, DelayedAudio(delay_ms, current_audio_delay_ms,
+ &extra_audio_delay_ms, &total_video_delay_ms));
+ EXPECT_EQ(0, extra_audio_delay_ms);
+ // The video delay is not allowed to change more than this in 1 second.
+ EXPECT_EQ(2*kMaxVideoDiffMs, total_video_delay_ms);
+
+ send_time_->IncreaseTimeMs(1000);
+ receive_time_->IncreaseTimeMs(800);
+ // Simulate 0 minimum delay in the VCM.
+ total_video_delay_ms = 0;
+ EXPECT_EQ(0, DelayedAudio(delay_ms, current_audio_delay_ms,
+ &extra_audio_delay_ms, &total_video_delay_ms));
+ EXPECT_EQ(0, extra_audio_delay_ms);
+ // The video delay is not allowed to change more than this in 1 second.
+ EXPECT_EQ(delay_ms, total_video_delay_ms);
+}
+
+TEST_F(StreamSynchronizationTest, AudioDelay) {
+ int current_audio_delay_ms = 0;
+ int delay_ms = 200;
+ int extra_audio_delay_ms = 0;
+ int current_extra_delay_ms = 0;
+ int total_video_delay_ms = 0;
+
+ EXPECT_EQ(0, DelayedVideo(delay_ms, current_audio_delay_ms,
+ &extra_audio_delay_ms, &total_video_delay_ms));
+ EXPECT_EQ(0, total_video_delay_ms);
+ // The audio delay is not allowed to change more than this in 1 second.
+ EXPECT_EQ(kMaxAudioDiffMs, extra_audio_delay_ms);
+ current_audio_delay_ms = extra_audio_delay_ms;
+ current_extra_delay_ms = extra_audio_delay_ms;
+
+ send_time_->IncreaseTimeMs(1000);
+ receive_time_->IncreaseTimeMs(800);
+ EXPECT_EQ(0, DelayedVideo(delay_ms, current_audio_delay_ms,
+ &extra_audio_delay_ms, &total_video_delay_ms));
+ EXPECT_EQ(0, total_video_delay_ms);
+ // The audio delay is not allowed to change more than the half of the required
+ // change in delay.
+ EXPECT_EQ(current_extra_delay_ms +
+ MaxAudioDelayIncrease(current_audio_delay_ms, delay_ms),
+ extra_audio_delay_ms);
+ current_audio_delay_ms = extra_audio_delay_ms;
+ current_extra_delay_ms = extra_audio_delay_ms;
+
+ send_time_->IncreaseTimeMs(1000);
+ receive_time_->IncreaseTimeMs(800);
+ EXPECT_EQ(0, DelayedVideo(delay_ms, current_audio_delay_ms,
+ &extra_audio_delay_ms, &total_video_delay_ms));
+ EXPECT_EQ(0, total_video_delay_ms);
+ // The audio delay is not allowed to change more than the half of the required
+ // change in delay.
+ EXPECT_EQ(current_extra_delay_ms +
+ MaxAudioDelayIncrease(current_audio_delay_ms, delay_ms),
+ extra_audio_delay_ms);
+ current_extra_delay_ms = extra_audio_delay_ms;
+
+ // Simulate that NetEQ for some reason reduced the delay.
+ current_audio_delay_ms = 170;
+ send_time_->IncreaseTimeMs(1000);
+ receive_time_->IncreaseTimeMs(800);
+ EXPECT_EQ(0, DelayedVideo(delay_ms, current_audio_delay_ms,
+ &extra_audio_delay_ms, &total_video_delay_ms));
+ EXPECT_EQ(0, total_video_delay_ms);
+ // Since we only can ask NetEQ for a certain amount of extra delay, and
+ // we only measure the total NetEQ delay, we will ask for additional delay
+ // here to try to
+ EXPECT_EQ(current_extra_delay_ms +
+ MaxAudioDelayIncrease(current_audio_delay_ms, delay_ms),
+ extra_audio_delay_ms);
+ current_extra_delay_ms = extra_audio_delay_ms;
+
+ // Simulate that NetEQ for some reason significantly increased the delay.
+ current_audio_delay_ms = 250;
+ send_time_->IncreaseTimeMs(1000);
+ receive_time_->IncreaseTimeMs(800);
+ EXPECT_EQ(0, DelayedVideo(delay_ms, current_audio_delay_ms,
+ &extra_audio_delay_ms, &total_video_delay_ms));
+ EXPECT_EQ(0, total_video_delay_ms);
+ // The audio delay is not allowed to change more than the half of the required
+ // change in delay.
+ EXPECT_EQ(current_extra_delay_ms +
+ MaxAudioDelayDecrease(current_audio_delay_ms, delay_ms),
+ extra_audio_delay_ms);
+}
+
+TEST_F(StreamSynchronizationTest, BothDelayedVideoLater) {
+ int current_audio_delay_ms = 0;
+ int audio_delay_ms = 100;
+ int video_delay_ms = 300;
+ int extra_audio_delay_ms = 0;
+ int current_extra_delay_ms = 0;
+ int total_video_delay_ms = 0;
+
+ EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
+ video_delay_ms,
+ current_audio_delay_ms,
+ &extra_audio_delay_ms,
+ &total_video_delay_ms));
+ EXPECT_EQ(0, total_video_delay_ms);
+ // The audio delay is not allowed to change more than this in 1 second.
+ EXPECT_EQ(kMaxAudioDiffMs, extra_audio_delay_ms);
+ current_audio_delay_ms = extra_audio_delay_ms;
+ current_extra_delay_ms = extra_audio_delay_ms;
+
+ send_time_->IncreaseTimeMs(1000);
+ receive_time_->IncreaseTimeMs(800);
+ EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
+ video_delay_ms,
+ current_audio_delay_ms,
+ &extra_audio_delay_ms,
+ &total_video_delay_ms));
+ EXPECT_EQ(0, total_video_delay_ms);
+ // The audio delay is not allowed to change more than the half of the required
+ // change in delay.
+ EXPECT_EQ(current_extra_delay_ms + MaxAudioDelayIncrease(
+ current_audio_delay_ms, video_delay_ms - audio_delay_ms),
+ extra_audio_delay_ms);
+ current_audio_delay_ms = extra_audio_delay_ms;
+ current_extra_delay_ms = extra_audio_delay_ms;
+
+ send_time_->IncreaseTimeMs(1000);
+ receive_time_->IncreaseTimeMs(800);
+ EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
+ video_delay_ms,
+ current_audio_delay_ms,
+ &extra_audio_delay_ms,
+ &total_video_delay_ms));
+ EXPECT_EQ(0, total_video_delay_ms);
+ // The audio delay is not allowed to change more than the half of the required
+ // change in delay.
+ EXPECT_EQ(current_extra_delay_ms + MaxAudioDelayIncrease(
+ current_audio_delay_ms, video_delay_ms - audio_delay_ms),
+ extra_audio_delay_ms);
+ current_extra_delay_ms = extra_audio_delay_ms;
+
+ // Simulate that NetEQ for some reason reduced the delay.
+ current_audio_delay_ms = 170;
+ send_time_->IncreaseTimeMs(1000);
+ receive_time_->IncreaseTimeMs(800);
+ EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
+ video_delay_ms,
+ current_audio_delay_ms,
+ &extra_audio_delay_ms,
+ &total_video_delay_ms));
+ EXPECT_EQ(0, total_video_delay_ms);
+ // Since we only can ask NetEQ for a certain amount of extra delay, and
+ // we only measure the total NetEQ delay, we will ask for additional delay
+ // here to try to stay in sync.
+ EXPECT_EQ(current_extra_delay_ms + MaxAudioDelayIncrease(
+ current_audio_delay_ms, video_delay_ms - audio_delay_ms),
+ extra_audio_delay_ms);
+ current_extra_delay_ms = extra_audio_delay_ms;
+
+ // Simulate that NetEQ for some reason significantly increased the delay.
+ current_audio_delay_ms = 250;
+ send_time_->IncreaseTimeMs(1000);
+ receive_time_->IncreaseTimeMs(800);
+ EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
+ video_delay_ms,
+ current_audio_delay_ms,
+ &extra_audio_delay_ms,
+ &total_video_delay_ms));
+ EXPECT_EQ(0, total_video_delay_ms);
+ // The audio delay is not allowed to change more than the half of the required
+ // change in delay.
+ EXPECT_EQ(current_extra_delay_ms + MaxAudioDelayIncrease(
+ current_audio_delay_ms, video_delay_ms - audio_delay_ms),
+ extra_audio_delay_ms);
+}
+
+TEST_F(StreamSynchronizationTest, BothDelayedAudioLater) {
+ int current_audio_delay_ms = 0;
+ int audio_delay_ms = 300;
+ int video_delay_ms = 100;
+ int extra_audio_delay_ms = 0;
+ int current_extra_delay_ms = 0;
+ int total_video_delay_ms = 0;
+
+ EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
+ video_delay_ms,
+ current_audio_delay_ms,
+ &extra_audio_delay_ms,
+ &total_video_delay_ms));
+ EXPECT_EQ(kMaxVideoDiffMs, total_video_delay_ms);
+ EXPECT_EQ(0, extra_audio_delay_ms);
+ current_audio_delay_ms = extra_audio_delay_ms;
+ current_extra_delay_ms = extra_audio_delay_ms;
+
+ send_time_->IncreaseTimeMs(1000);
+ receive_time_->IncreaseTimeMs(1000 - std::max(audio_delay_ms,
+ video_delay_ms));
+ // Simulate 0 minimum delay in the VCM.
+ total_video_delay_ms = 0;
+ EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
+ video_delay_ms,
+ current_audio_delay_ms,
+ &extra_audio_delay_ms,
+ &total_video_delay_ms));
+ EXPECT_EQ(2 * kMaxVideoDiffMs, total_video_delay_ms);
+ EXPECT_EQ(0, extra_audio_delay_ms);
+ current_audio_delay_ms = extra_audio_delay_ms;
+ current_extra_delay_ms = extra_audio_delay_ms;
+
+ send_time_->IncreaseTimeMs(1000);
+ receive_time_->IncreaseTimeMs(1000 - std::max(audio_delay_ms,
+ video_delay_ms));
+ // Simulate 0 minimum delay in the VCM.
+ total_video_delay_ms = 0;
+ EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
+ video_delay_ms,
+ current_audio_delay_ms,
+ &extra_audio_delay_ms,
+ &total_video_delay_ms));
+ EXPECT_EQ(audio_delay_ms - video_delay_ms, total_video_delay_ms);
+ EXPECT_EQ(0, extra_audio_delay_ms);
+ current_extra_delay_ms = extra_audio_delay_ms;
+
+ // Simulate that NetEQ introduces some audio delay.
+ current_audio_delay_ms = 50;
+ send_time_->IncreaseTimeMs(1000);
+ receive_time_->IncreaseTimeMs(1000 - std::max(audio_delay_ms,
+ video_delay_ms));
+ // Simulate 0 minimum delay in the VCM.
+ total_video_delay_ms = 0;
+ EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
+ video_delay_ms,
+ current_audio_delay_ms,
+ &extra_audio_delay_ms,
+ &total_video_delay_ms));
+ EXPECT_EQ(audio_delay_ms - video_delay_ms + current_audio_delay_ms,
+ total_video_delay_ms);
+ EXPECT_EQ(0, extra_audio_delay_ms);
+ current_extra_delay_ms = extra_audio_delay_ms;
+
+ // Simulate that NetEQ reduces its delay.
+ current_audio_delay_ms = 10;
+ send_time_->IncreaseTimeMs(1000);
+ receive_time_->IncreaseTimeMs(1000 - std::max(audio_delay_ms,
+ video_delay_ms));
+ // Simulate 0 minimum delay in the VCM.
+ total_video_delay_ms = 0;
+ EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
+ video_delay_ms,
+ current_audio_delay_ms,
+ &extra_audio_delay_ms,
+ &total_video_delay_ms));
+ EXPECT_EQ(audio_delay_ms - video_delay_ms + current_audio_delay_ms,
+ total_video_delay_ms);
+ EXPECT_EQ(0, extra_audio_delay_ms);
+}
+} // namespace webrtc
diff --git a/src/video_engine/video_engine_core.gypi b/src/video_engine/video_engine_core.gypi
index cf4a1ec..10c44fc 100644
--- a/src/video_engine/video_engine_core.gypi
+++ b/src/video_engine/video_engine_core.gypi
@@ -62,6 +62,7 @@
'include/vie_rtp_rtcp.h',
# headers
+ 'stream_synchronization.h',
'vie_base_impl.h',
'vie_capture_impl.h',
'vie_codec_impl.h',
@@ -96,6 +97,7 @@
'vie_sync_module.h',
# ViE
+ 'stream_synchronization.cc',
'vie_base_impl.cc',
'vie_capture_impl.cc',
'vie_codec_impl.cc',
@@ -148,6 +150,7 @@
'../modules/rtp_rtcp/interface',
],
'sources': [
+ 'stream_synchronization_unittest.cc',
'vie_remb_unittest.cc',
],
},
diff --git a/src/video_engine/vie_sync_module.cc b/src/video_engine/vie_sync_module.cc
index 5aa5595..01aa5d2 100644
--- a/src/video_engine/vie_sync_module.cc
+++ b/src/video_engine/vie_sync_module.cc
@@ -15,15 +15,11 @@
#include "trace.h"
#include "video_coding.h"
#include "voe_video_sync.h"
+#include "video_engine/stream_synchronization.h"
namespace webrtc {
enum { kSyncInterval = 1000};
-enum { kMaxVideoDiffMs = 80 };
-enum { kMaxAudioDiffMs = 80 };
-enum { kMaxDelay = 1500 };
-
-const float FracMS = 4.294967296E6f;
ViESyncModule::ViESyncModule(const int32_t channel_id, VideoCodingModule* vcm)
: data_cs_(CriticalSectionWrapper::CreateCriticalSection()),
@@ -32,7 +28,8 @@
video_rtcp_module_(NULL),
voe_channel_id_(-1),
voe_sync_interface_(NULL),
- last_sync_time_(TickTime::Now()) {
+ last_sync_time_(TickTime::Now()),
+ sync_() {
}
ViESyncModule::~ViESyncModule() {
@@ -45,6 +42,7 @@
voe_channel_id_ = voe_channel_id;
voe_sync_interface_ = voe_sync_interface;
video_rtcp_module_ = video_rtcp_module;
+ sync_.reset(new StreamSynchronization(voe_channel_id, channel_id_));
if (!voe_sync_interface) {
voe_channel_id_ = -1;
@@ -62,7 +60,7 @@
}
WebRtc_Word32 ViESyncModule::TimeUntilNextProcess() {
- return (WebRtc_Word32)(kSyncInterval -
+ return static_cast<WebRtc_Word32>(kSyncInterval -
(TickTime::Now() - last_sync_time_).Milliseconds());
}
@@ -79,6 +77,7 @@
return 0;
}
assert(video_rtcp_module_ && voe_sync_interface_);
+ assert(sync_.get());
int current_audio_delay_ms = 0;
if (voe_sync_interface_->GetDelayEstimate(voe_channel_id_,
@@ -90,9 +89,6 @@
return 0;
}
- int current_diff_ms = 0;
- // Total video delay.
- int video_delay_ms = 0;
// VoiceEngine report delay estimates even when not started, ignore if the
// reported value is lower than 40 ms.
if (current_audio_delay_ms < 40) {
@@ -108,211 +104,34 @@
}
assert(voice_rtcp_module);
- uint32_t video_received_ntp_secs = 0;
- uint32_t video_received_ntp_frac = 0;
- uint32_t video_rtcp_arrivaltime_secs = 0;
- uint32_t video_rtcp_arrivaltime_frac = 0;
-
- if (0 != video_rtcp_module_->RemoteNTP(&video_received_ntp_secs,
- &video_received_ntp_frac,
- &video_rtcp_arrivaltime_secs,
- &video_rtcp_arrivaltime_frac)) {
+ StreamSynchronization::Measurements video;
+ if (0 != video_rtcp_module_->RemoteNTP(&video.received_ntp_secs,
+ &video.received_ntp_frac,
+ &video.rtcp_arrivaltime_secs,
+ &video.rtcp_arrivaltime_frac)) {
// Failed to get video NTP.
return 0;
}
- uint32_t audio_received_ntp_secs = 0;
- uint32_t audio_received_ntp_frac = 0;
- uint32_t audio_rtcp_arrivaltime_secs = 0;
- uint32_t audio_rtcp_arrivaltime_frac = 0;
- if (0 != voice_rtcp_module->RemoteNTP(&audio_received_ntp_secs,
- &audio_received_ntp_frac,
- &audio_rtcp_arrivaltime_secs,
- &audio_rtcp_arrivaltime_frac)) {
+ StreamSynchronization::Measurements audio;
+ if (0 != voice_rtcp_module->RemoteNTP(&audio.received_ntp_secs,
+ &audio.received_ntp_frac,
+ &audio.rtcp_arrivaltime_secs,
+ &audio.rtcp_arrivaltime_frac)) {
// Failed to get audio NTP.
return 0;
}
- // ReceivedNTPxxx is NTP at sender side when sent.
- // RTCPArrivalTimexxx is NTP at receiver side when received.
- // can't use ConvertNTPTimeToMS since calculation can be
- // negative
- int NTPdiff = (audio_received_ntp_secs - video_received_ntp_secs)
- * 1000; // ms
- NTPdiff += static_cast<int>(audio_received_ntp_frac / FracMS -
- video_received_ntp_frac / FracMS);
-
- int RTCPdiff = (audio_rtcp_arrivaltime_secs - video_rtcp_arrivaltime_secs)
- * 1000; // ms
- RTCPdiff += static_cast<int>(audio_rtcp_arrivaltime_frac / FracMS -
- video_rtcp_arrivaltime_frac / FracMS);
-
- int diff = NTPdiff - RTCPdiff;
- // if diff is + video is behind
- if (diff < -1000 || diff > 1000) {
- // unresonable ignore value.
+ int extra_audio_delay_ms = 0;
+ if (sync_->ComputeDelays(audio, current_audio_delay_ms, &extra_audio_delay_ms,
+ video, &total_video_delay_target_ms) != 0) {
return 0;
}
- channel_delay_.network_delay = diff;
-
- WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, channel_id_,
- "Audio delay is: %d for voice channel: %d",
- current_audio_delay_ms, voe_channel_id_);
- WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, channel_id_,
- "Network delay diff is: %d for voice channel: %d",
- channel_delay_.network_delay, voe_channel_id_);
- // Calculate the difference between the lowest possible video delay and
- // the current audio delay.
- current_diff_ms = total_video_delay_target_ms - current_audio_delay_ms +
- channel_delay_.network_delay;
- WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, channel_id_,
- "Current diff is: %d for audio channel: %d",
- current_diff_ms, voe_channel_id_);
-
- if (current_diff_ms > 0) {
- // The minimum video delay is longer than the current audio delay.
- // We need to decrease extra video delay, if we have added extra delay
- // earlier, or add extra audio delay.
- if (channel_delay_.extra_video_delay_ms > 0) {
- // We have extra delay added to ViE. Reduce this delay before adding
- // extra delay to VoE.
-
- // This is the desired delay, we can't reduce more than this.
- video_delay_ms = total_video_delay_target_ms;
-
- // Check that we don't reduce the delay more than what is allowed.
- if (video_delay_ms <
- channel_delay_.last_video_delay_ms - kMaxVideoDiffMs) {
- video_delay_ms =
- channel_delay_.last_video_delay_ms - kMaxVideoDiffMs;
- channel_delay_.extra_video_delay_ms =
- video_delay_ms - total_video_delay_target_ms;
- } else {
- channel_delay_.extra_video_delay_ms = 0;
- }
- channel_delay_.last_video_delay_ms = video_delay_ms;
- channel_delay_.last_sync_delay = -1;
- channel_delay_.extra_audio_delay_ms = 0;
- } else { // channel_delay_.extra_video_delay_ms > 0
- // We have no extra video delay to remove, increase the audio delay.
- if (channel_delay_.last_sync_delay >= 0) {
- // We have increased the audio delay earlier, increase it even more.
- int audio_diff_ms = current_diff_ms / 2;
- if (audio_diff_ms > kMaxAudioDiffMs) {
- // We only allow a maximum change of KMaxAudioDiffMS for audio
- // due to NetEQ maximum changes.
- audio_diff_ms = kMaxAudioDiffMs;
- }
- // Increase the audio delay
- channel_delay_.extra_audio_delay_ms += audio_diff_ms;
-
- // Don't set a too high delay.
- if (channel_delay_.extra_audio_delay_ms > kMaxDelay) {
- channel_delay_.extra_audio_delay_ms = kMaxDelay;
- }
-
- // Don't add any extra video delay.
- video_delay_ms = total_video_delay_target_ms;
- channel_delay_.extra_video_delay_ms = 0;
- channel_delay_.last_video_delay_ms = video_delay_ms;
- channel_delay_.last_sync_delay = 1;
- } else { // channel_delay_.last_sync_delay >= 0
- // First time after a delay change, don't add any extra delay.
- // This is to not toggle back and forth too much.
- channel_delay_.extra_audio_delay_ms = 0;
- // Set minimum video delay
- video_delay_ms = total_video_delay_target_ms;
- channel_delay_.extra_video_delay_ms = 0;
- channel_delay_.last_video_delay_ms = video_delay_ms;
- channel_delay_.last_sync_delay = 0;
- }
- }
- } else { // if (current_diffMS > 0)
- // The minimum video delay is lower than the current audio delay.
- // We need to decrease possible extra audio delay, or
- // add extra video delay.
-
- if (channel_delay_.extra_audio_delay_ms > 0) {
- // We have extra delay in VoiceEngine
- // Start with decreasing the voice delay
- int audio_diff_ms = current_diff_ms / 2;
- if (audio_diff_ms < -1 * kMaxAudioDiffMs) {
- // Don't change the delay too much at once.
- audio_diff_ms = -1 * kMaxAudioDiffMs;
- }
- // Add the negative difference.
- channel_delay_.extra_audio_delay_ms += audio_diff_ms;
-
- if (channel_delay_.extra_audio_delay_ms < 0) {
- // Negative values not allowed.
- channel_delay_.extra_audio_delay_ms = 0;
- channel_delay_.last_sync_delay = 0;
- } else {
- // There is more audio delay to use for the next round.
- channel_delay_.last_sync_delay = 1;
- }
-
- // Keep the video delay at the minimum values.
- video_delay_ms = total_video_delay_target_ms;
- channel_delay_.extra_video_delay_ms = 0;
- channel_delay_.last_video_delay_ms = video_delay_ms;
- } else { // channel_delay_.extra_audio_delay_ms > 0
- // We have no extra delay in VoiceEngine, increase the video delay.
- channel_delay_.extra_audio_delay_ms = 0;
-
- // Make the difference positive.
- int video_diff_ms = -1 * current_diff_ms;
-
- // This is the desired delay.
- video_delay_ms = total_video_delay_target_ms + video_diff_ms;
- if (video_delay_ms > channel_delay_.last_video_delay_ms) {
- if (video_delay_ms >
- channel_delay_.last_video_delay_ms + kMaxVideoDiffMs) {
- // Don't increase the delay too much at once
- video_delay_ms =
- channel_delay_.last_video_delay_ms + kMaxVideoDiffMs;
- }
- // Verify we don't go above the maximum allowed delay
- if (video_delay_ms > kMaxDelay) {
- video_delay_ms = kMaxDelay;
- }
- } else {
- if (video_delay_ms <
- channel_delay_.last_video_delay_ms - kMaxVideoDiffMs) {
- // Don't decrease the delay too much at once
- video_delay_ms =
- channel_delay_.last_video_delay_ms - kMaxVideoDiffMs;
- }
- // Verify we don't go below the minimum delay
- if (video_delay_ms < total_video_delay_target_ms) {
- video_delay_ms = total_video_delay_target_ms;
- }
- }
- // Store the values
- channel_delay_.extra_video_delay_ms =
- video_delay_ms - total_video_delay_target_ms;
- channel_delay_.last_video_delay_ms = video_delay_ms;
- channel_delay_.last_sync_delay = -1;
- }
- }
-
- WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, channel_id_,
- "Sync video delay %d ms for video channel and audio delay %d for audio "
- "channel %d",
- video_delay_ms, channel_delay_.extra_audio_delay_ms, voe_channel_id_);
-
// Set the extra audio delay.synchronization
if (voe_sync_interface_->SetMinimumPlayoutDelay(
- voe_channel_id_, channel_delay_.extra_audio_delay_ms) == -1) {
+ voe_channel_id_, extra_audio_delay_ms) == -1) {
WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideo, channel_id_,
"Error setting voice delay");
}
-
- if (video_delay_ms < 0) {
- video_delay_ms = 0;
- }
- total_video_delay_target_ms =
- (total_video_delay_target_ms > video_delay_ms) ?
- total_video_delay_target_ms : video_delay_ms;
vcm_->SetMinimumPlayoutDelay(total_video_delay_target_ms);
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, channel_id_,
"New Video delay target is: %d", total_video_delay_target_ms);
diff --git a/src/video_engine/vie_sync_module.h b/src/video_engine/vie_sync_module.h
index 052f7e3..c93d586 100644
--- a/src/video_engine/vie_sync_module.h
+++ b/src/video_engine/vie_sync_module.h
@@ -14,14 +14,15 @@
#ifndef WEBRTC_VIDEO_ENGINE_VIE_SYNC_MODULE_H_
#define WEBRTC_VIDEO_ENGINE_VIE_SYNC_MODULE_H_
-#include "module.h"
+#include "modules/interface/module.h"
#include "system_wrappers/interface/scoped_ptr.h"
-#include "tick_util.h"
+#include "system_wrappers/interface/tick_util.h"
namespace webrtc {
class CriticalSectionWrapper;
class RtpRtcp;
+class StreamSynchronization;
class VideoCodingModule;
class VoEVideoSync;
@@ -48,22 +49,7 @@
int voe_channel_id_;
VoEVideoSync* voe_sync_interface_;
TickTime last_sync_time_;
-
- struct ViESyncDelay {
- ViESyncDelay() {
- extra_video_delay_ms = 0;
- last_video_delay_ms = 0;
- extra_audio_delay_ms = 0;
- last_sync_delay = 0;
- network_delay = 120;
- }
- int extra_video_delay_ms;
- int last_video_delay_ms;
- int extra_audio_delay_ms;
- int last_sync_delay;
- int network_delay;
- };
- ViESyncDelay channel_delay_;
+ scoped_ptr<StreamSynchronization> sync_;
};
} // namespace webrtc