Updated the sync module with a slow moving filter
Review URL: https://webrtc-codereview.appspot.com/1326008
git-svn-id: http://webrtc.googlecode.com/svn/trunk@3884 4adac7df-926f-26a2-2b94-8c16560cd09d
diff --git a/webrtc/video_engine/stream_synchronization.cc b/webrtc/video_engine/stream_synchronization.cc
index 48eb36a..3e8d0ea 100644
--- a/webrtc/video_engine/stream_synchronization.cc
+++ b/webrtc/video_engine/stream_synchronization.cc
@@ -18,9 +18,11 @@
namespace webrtc {
-const int kMaxVideoDiffMs = 80;
-const int kMaxAudioDiffMs = 80;
-const int kMaxDeltaDelayMs = 1500;
+static const int kMaxChangeMs = 80;
+static const int kMaxDeltaDelayMs = 10000;
+static const int kFilterLength = 4;
+// Minimum difference between audio and video to warrant a change.
+static const int kMinDeltaMs = 30;
struct ViESyncDelay {
ViESyncDelay() {
@@ -43,7 +45,8 @@
: channel_delay_(new ViESyncDelay),
audio_channel_id_(audio_channel_id),
video_channel_id_(video_channel_id),
- base_target_delay_ms_(0) {}
+ base_target_delay_ms_(0),
+ avg_diff_ms_(0) {}
StreamSynchronization::~StreamSynchronization() {
delete channel_delay_;
@@ -100,11 +103,24 @@
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_,
"Current diff is: %d for audio channel: %d",
relative_delay_ms, audio_channel_id_);
+
int current_diff_ms = *total_video_delay_target_ms - current_audio_delay_ms +
relative_delay_ms;
+ avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ +
+ current_diff_ms) / kFilterLength;
+ if (abs(avg_diff_ms_) < kMinDeltaMs) {
+ // Don't adjust if the diff is within our margin.
+ return false;
+ }
+
+ // Make sure we don't move too fast.
+ int diff_ms = avg_diff_ms_ / 2;
+ diff_ms = std::min(diff_ms, kMaxChangeMs);
+ diff_ms = std::max(diff_ms, -kMaxChangeMs);
+
int video_delay_ms = base_target_delay_ms_;
- if (current_diff_ms > 0) {
+ if (diff_ms > 0) {
// The minimum video delay is longer than the current audio delay.
// We need to decrease extra video delay, if we have added extra delay
// earlier, or add extra audio delay.
@@ -116,10 +132,8 @@
video_delay_ms = *total_video_delay_target_ms;
// Check that we don't reduce the delay more than what is allowed.
- if (video_delay_ms <
- channel_delay_->last_video_delay_ms - kMaxVideoDiffMs) {
- video_delay_ms =
- channel_delay_->last_video_delay_ms - kMaxVideoDiffMs;
+ if (video_delay_ms < channel_delay_->last_video_delay_ms - diff_ms) {
+ video_delay_ms = channel_delay_->last_video_delay_ms - diff_ms;
channel_delay_->extra_video_delay_ms =
video_delay_ms - *total_video_delay_target_ms;
} else {
@@ -132,21 +146,13 @@
// We have no extra video delay to remove, increase the audio delay.
if (channel_delay_->last_sync_delay >= 0) {
// We have increased the audio delay earlier, increase it even more.
- int audio_diff_ms = current_diff_ms / 2;
- if (audio_diff_ms > kMaxAudioDiffMs) {
- // We only allow a maximum change of KMaxAudioDiffMS for audio
- // due to NetEQ maximum changes.
- audio_diff_ms = kMaxAudioDiffMs;
- }
// Increase the audio delay.
- channel_delay_->extra_audio_delay_ms += audio_diff_ms;
+ channel_delay_->extra_audio_delay_ms += diff_ms;
// Don't set a too high delay.
- if (channel_delay_->extra_audio_delay_ms >
- base_target_delay_ms_ + kMaxDeltaDelayMs) {
- channel_delay_->extra_audio_delay_ms =
- base_target_delay_ms_ + kMaxDeltaDelayMs;
- }
+ channel_delay_->extra_audio_delay_ms = std::min(
+ channel_delay_->extra_audio_delay_ms,
+ base_target_delay_ms_ + kMaxDeltaDelayMs);
// Don't add any extra video delay.
video_delay_ms = *total_video_delay_target_ms;
@@ -164,20 +170,15 @@
channel_delay_->last_sync_delay = 0;
}
}
- } else { // if (current_diff_ms > 0)
+ } else { // if (diff_ms > 0)
// The minimum video delay is lower than the current audio delay.
// We need to decrease possible extra audio delay, or
// add extra video delay.
if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) {
// We have extra delay in VoiceEngine.
// Start with decreasing the voice delay.
- int audio_diff_ms = current_diff_ms / 2;
- if (audio_diff_ms < -1 * kMaxAudioDiffMs) {
- // Don't change the delay too much at once.
- audio_diff_ms = -1 * kMaxAudioDiffMs;
- }
- // Add the negative difference.
- channel_delay_->extra_audio_delay_ms += audio_diff_ms;
+ // Note: diff_ms is negative; add the negative difference.
+ channel_delay_->extra_audio_delay_ms += diff_ms;
if (channel_delay_->extra_audio_delay_ms < 0) {
// Negative values not allowed.
@@ -196,34 +197,15 @@
// We have no extra delay in VoiceEngine, increase the video delay.
channel_delay_->extra_audio_delay_ms = base_target_delay_ms_;
- // Make the difference positive.
- int video_diff_ms = -1 * current_diff_ms;
-
// This is the desired delay.
- video_delay_ms = *total_video_delay_target_ms + video_diff_ms;
- if (video_delay_ms > channel_delay_->last_video_delay_ms) {
- if (video_delay_ms >
- channel_delay_->last_video_delay_ms + kMaxVideoDiffMs) {
- // Don't increase the delay too much at once.
- video_delay_ms =
- channel_delay_->last_video_delay_ms + kMaxVideoDiffMs;
- }
- // Verify we don't go above the maximum allowed delay.
- if (video_delay_ms > base_target_delay_ms_ + kMaxDeltaDelayMs) {
- video_delay_ms = base_target_delay_ms_ + kMaxDeltaDelayMs;
- }
- } else {
- if (video_delay_ms <
- channel_delay_->last_video_delay_ms - kMaxVideoDiffMs) {
- // Don't decrease the delay too much at once.
- video_delay_ms =
- channel_delay_->last_video_delay_ms - kMaxVideoDiffMs;
- }
- // Verify we don't go below the minimum delay.
- if (video_delay_ms < *total_video_delay_target_ms) {
- video_delay_ms = *total_video_delay_target_ms;
- }
- }
+ // Note: diff_ms is negative.
+ video_delay_ms = std::max(*total_video_delay_target_ms,
+ channel_delay_->last_video_delay_ms) - diff_ms;
+
+ // Verify we don't go above the maximum allowed delay.
+ video_delay_ms = std::min(video_delay_ms,
+ base_target_delay_ms_ + kMaxDeltaDelayMs);
+
// Store the values.
channel_delay_->extra_video_delay_ms =
video_delay_ms - *total_video_delay_target_ms;
@@ -231,20 +213,16 @@
channel_delay_->last_sync_delay = -1;
}
}
-
+ avg_diff_ms_ = 0;
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_,
"Sync video delay %d ms for video channel and audio delay %d for audio "
"channel %d",
video_delay_ms, channel_delay_->extra_audio_delay_ms, audio_channel_id_);
*extra_audio_delay_ms = channel_delay_->extra_audio_delay_ms;
-
- if (video_delay_ms < 0) {
- video_delay_ms = 0;
- }
- *total_video_delay_target_ms =
- (*total_video_delay_target_ms > video_delay_ms) ?
- *total_video_delay_target_ms : video_delay_ms;
+ video_delay_ms = std::max(video_delay_ms, 0);
+ *total_video_delay_target_ms = std::max(*total_video_delay_target_ms,
+ video_delay_ms);
return true;
}
diff --git a/webrtc/video_engine/stream_synchronization.h b/webrtc/video_engine/stream_synchronization.h
index 9b7780c..f057b76 100644
--- a/webrtc/video_engine/stream_synchronization.h
+++ b/webrtc/video_engine/stream_synchronization.h
@@ -52,6 +52,7 @@
int audio_channel_id_;
int video_channel_id_;
int base_target_delay_ms_;
+ int avg_diff_ms_;
};
} // namespace webrtc
diff --git a/webrtc/video_engine/stream_synchronization_unittest.cc b/webrtc/video_engine/stream_synchronization_unittest.cc
index 44cb146..395556b 100644
--- a/webrtc/video_engine/stream_synchronization_unittest.cc
+++ b/webrtc/video_engine/stream_synchronization_unittest.cc
@@ -25,6 +25,7 @@
enum { kDefaultAudioFrequency = 8000 };
enum { kDefaultVideoFrequency = 90000 };
const double kNtpFracPerMs = 4.294967296E6;
+static const int kSmoothingFilter = 4 * 2;
class Time {
public:
@@ -160,13 +161,14 @@
int video_delay_ms = base_target_delay + 100;
int extra_audio_delay_ms = 0;
int total_video_delay_ms = base_target_delay;
+ int filtered_move = (audio_delay_ms - video_delay_ms) / kSmoothingFilter;
EXPECT_TRUE(DelayedStreams(audio_delay_ms,
video_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
- EXPECT_EQ(base_target_delay + kMaxVideoDiffMs, total_video_delay_ms);
+ EXPECT_EQ(base_target_delay + filtered_move, total_video_delay_ms);
EXPECT_EQ(base_target_delay, extra_audio_delay_ms);
current_audio_delay_ms = extra_audio_delay_ms;
@@ -180,7 +182,7 @@
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
- EXPECT_EQ(base_target_delay + 2 * kMaxVideoDiffMs, total_video_delay_ms);
+ EXPECT_EQ(base_target_delay + 2 * filtered_move, total_video_delay_ms);
EXPECT_EQ(base_target_delay, extra_audio_delay_ms);
current_audio_delay_ms = extra_audio_delay_ms;
@@ -194,8 +196,7 @@
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
- EXPECT_EQ(base_target_delay + audio_delay_ms - video_delay_ms,
- total_video_delay_ms);
+ EXPECT_EQ(base_target_delay + 3 * filtered_move, total_video_delay_ms);
EXPECT_EQ(base_target_delay, extra_audio_delay_ms);
// Simulate that NetEQ introduces some audio delay.
@@ -210,8 +211,9 @@
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
- EXPECT_EQ(audio_delay_ms - video_delay_ms + current_audio_delay_ms,
- total_video_delay_ms);
+ filtered_move = 3 * filtered_move +
+ (50 + audio_delay_ms - video_delay_ms) / kSmoothingFilter;
+ EXPECT_EQ(base_target_delay + filtered_move, total_video_delay_ms);
EXPECT_EQ(base_target_delay, extra_audio_delay_ms);
// Simulate that NetEQ reduces its delay.
@@ -226,8 +228,11 @@
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
- EXPECT_EQ(audio_delay_ms - video_delay_ms + current_audio_delay_ms,
- total_video_delay_ms);
+
+ filtered_move = filtered_move +
+ (10 + audio_delay_ms - video_delay_ms) / kSmoothingFilter;
+
+ EXPECT_EQ(base_target_delay + filtered_move, total_video_delay_ms);
EXPECT_EQ(base_target_delay, extra_audio_delay_ms);
}
@@ -245,7 +250,7 @@
&total_video_delay_ms));
EXPECT_EQ(base_target_delay, total_video_delay_ms);
// The audio delay is not allowed to change more than this in 1 second.
- EXPECT_EQ(base_target_delay + kMaxAudioDiffMs, extra_audio_delay_ms);
+ EXPECT_GE(base_target_delay + kMaxAudioDiffMs, extra_audio_delay_ms);
current_audio_delay_ms = extra_audio_delay_ms;
int current_extra_delay_ms = extra_audio_delay_ms;
@@ -283,7 +288,7 @@
current_extra_delay_ms = extra_audio_delay_ms;
// Simulate that NetEQ for some reason reduced the delay.
- current_audio_delay_ms = base_target_delay + 170;
+ current_audio_delay_ms = base_target_delay + 10;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_TRUE(DelayedStreams(audio_delay_ms,
@@ -302,7 +307,7 @@
current_extra_delay_ms = extra_audio_delay_ms;
// Simulate that NetEQ for some reason significantly increased the delay.
- current_audio_delay_ms = base_target_delay + 250;
+ current_audio_delay_ms = base_target_delay + 350;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_TRUE(DelayedStreams(audio_delay_ms,
@@ -320,12 +325,13 @@
}
int MaxAudioDelayIncrease(int current_audio_delay_ms, int delay_ms) {
- return std::min((delay_ms - current_audio_delay_ms) / 2,
+ return std::min((delay_ms - current_audio_delay_ms) / kSmoothingFilter,
static_cast<int>(kMaxAudioDiffMs));
}
int MaxAudioDelayDecrease(int current_audio_delay_ms, int delay_ms) {
- return std::max((delay_ms - current_audio_delay_ms) / 2, -kMaxAudioDiffMs);
+ return std::max((delay_ms - current_audio_delay_ms) / kSmoothingFilter,
+ -kMaxAudioDiffMs);
}
enum { kSendTimeOffsetMs = 98765 };
@@ -343,8 +349,8 @@
int extra_audio_delay_ms = 0;
int total_video_delay_ms = 0;
- EXPECT_TRUE(DelayedStreams(0, 0, current_audio_delay_ms,
- &extra_audio_delay_ms, &total_video_delay_ms));
+ EXPECT_FALSE(DelayedStreams(0, 0, current_audio_delay_ms,
+ &extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, extra_audio_delay_ms);
EXPECT_EQ(0, total_video_delay_ms);
}
@@ -359,7 +365,7 @@
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, extra_audio_delay_ms);
// The video delay is not allowed to change more than this in 1 second.
- EXPECT_EQ(kMaxVideoDiffMs, total_video_delay_ms);
+ EXPECT_EQ(delay_ms / kSmoothingFilter, total_video_delay_ms);
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
@@ -369,7 +375,7 @@
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, extra_audio_delay_ms);
// The video delay is not allowed to change more than this in 1 second.
- EXPECT_EQ(2*kMaxVideoDiffMs, total_video_delay_ms);
+ EXPECT_EQ(2 * delay_ms / kSmoothingFilter, total_video_delay_ms);
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
@@ -380,7 +386,7 @@
EXPECT_EQ(0, extra_audio_delay_ms);
// Enough time should have elapsed for the requested total video delay to be
// equal to the relative delay between audio and video, i.e., we are in sync.
- EXPECT_EQ(delay_ms, total_video_delay_ms);
+ EXPECT_EQ(3 * delay_ms / kSmoothingFilter, total_video_delay_ms);
}
TEST_F(StreamSynchronizationTest, AudioDelay) {
@@ -393,7 +399,7 @@
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, total_video_delay_ms);
// The audio delay is not allowed to change more than this in 1 second.
- EXPECT_EQ(kMaxAudioDiffMs, extra_audio_delay_ms);
+ EXPECT_EQ(delay_ms / kSmoothingFilter, extra_audio_delay_ms);
current_audio_delay_ms = extra_audio_delay_ms;
int current_extra_delay_ms = extra_audio_delay_ms;
@@ -423,7 +429,7 @@
current_extra_delay_ms = extra_audio_delay_ms;
// Simulate that NetEQ for some reason reduced the delay.
- current_audio_delay_ms = 170;
+ current_audio_delay_ms = 10;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_TRUE(DelayedStreams(0, delay_ms, current_audio_delay_ms,
@@ -438,7 +444,7 @@
current_extra_delay_ms = extra_audio_delay_ms;
// Simulate that NetEQ for some reason significantly increased the delay.
- current_audio_delay_ms = 250;
+ current_audio_delay_ms = 350;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_TRUE(DelayedStreams(0, delay_ms, current_audio_delay_ms,
@@ -485,32 +491,29 @@
int extra_audio_delay_ms = 0;
int total_video_delay_ms = base_target_delay_ms;
sync_->SetTargetBufferingDelay(base_target_delay_ms);
- EXPECT_TRUE(DelayedStreams(base_target_delay_ms, base_target_delay_ms,
- current_audio_delay_ms,
- &extra_audio_delay_ms, &total_video_delay_ms));
- EXPECT_EQ(base_target_delay_ms, extra_audio_delay_ms);
- EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
+ // We are in sync don't change.
+ EXPECT_FALSE(DelayedStreams(base_target_delay_ms, base_target_delay_ms,
+ current_audio_delay_ms,
+ &extra_audio_delay_ms, &total_video_delay_ms));
// Triggering another call with the same values. Delay should not be modified.
base_target_delay_ms = 2000;
current_audio_delay_ms = base_target_delay_ms;
total_video_delay_ms = base_target_delay_ms;
sync_->SetTargetBufferingDelay(base_target_delay_ms);
- EXPECT_TRUE(DelayedStreams(base_target_delay_ms, base_target_delay_ms,
- current_audio_delay_ms,
- &extra_audio_delay_ms, &total_video_delay_ms));
- EXPECT_EQ(base_target_delay_ms, extra_audio_delay_ms);
- EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
+ // We are in sync don't change.
+ EXPECT_FALSE(DelayedStreams(base_target_delay_ms, base_target_delay_ms,
+ current_audio_delay_ms,
+ &extra_audio_delay_ms, &total_video_delay_ms));
// Changing delay value - intended to test this module only. In practice it
// would take VoE time to adapt.
base_target_delay_ms = 5000;
current_audio_delay_ms = base_target_delay_ms;
total_video_delay_ms = base_target_delay_ms;
sync_->SetTargetBufferingDelay(base_target_delay_ms);
- EXPECT_TRUE(DelayedStreams(base_target_delay_ms, base_target_delay_ms,
- current_audio_delay_ms,
- &extra_audio_delay_ms, &total_video_delay_ms));
- EXPECT_EQ(base_target_delay_ms, extra_audio_delay_ms);
- EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
+ // We are in sync don't change.
+ EXPECT_FALSE(DelayedStreams(base_target_delay_ms, base_target_delay_ms,
+ current_audio_delay_ms,
+ &extra_audio_delay_ms, &total_video_delay_ms));
}
TEST_F(StreamSynchronizationTest, BothDelayedAudioLaterWithBaseDelay) {