Stop using VoEVideoSync in Call/VideoReceiveStream.
BUG=webrtc:4690
Review-Url: https://codereview.webrtc.org/2452163004
Cr-Commit-Position: refs/heads/master@{#16375}
diff --git a/webrtc/audio/audio_receive_stream.cc b/webrtc/audio/audio_receive_stream.cc
index aaf77a8..1f24b2c 100644
--- a/webrtc/audio/audio_receive_stream.cc
+++ b/webrtc/audio/audio_receive_stream.cc
@@ -21,6 +21,8 @@
#include "webrtc/base/logging.h"
#include "webrtc/base/timeutils.h"
#include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
+#include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h"
+#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
#include "webrtc/voice_engine/channel_proxy.h"
#include "webrtc/voice_engine/include/voe_base.h"
#include "webrtc/voice_engine/include/voe_codec.h"
@@ -81,6 +83,8 @@
RTC_DCHECK(remote_bitrate_estimator);
RTC_DCHECK(rtp_header_parser_);
+ module_process_thread_checker_.DetachFromThread();
+
VoiceEngineImpl* voe_impl = static_cast<VoiceEngineImpl*>(voice_engine());
channel_proxy_ = voe_impl->GetChannelProxy(config_.voe_channel_id);
channel_proxy_->SetRtcEventLog(event_log);
@@ -125,7 +129,7 @@
}
AudioReceiveStream::~AudioReceiveStream() {
- RTC_DCHECK_RUN_ON(&thread_checker_);
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
LOG(LS_INFO) << "~AudioReceiveStream: " << config_.ToString();
if (playing_) {
Stop();
@@ -138,7 +142,7 @@
}
void AudioReceiveStream::Start() {
- RTC_DCHECK_RUN_ON(&thread_checker_);
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
if (playing_) {
return;
}
@@ -159,7 +163,7 @@
}
void AudioReceiveStream::Stop() {
- RTC_DCHECK_RUN_ON(&thread_checker_);
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
if (!playing_) {
return;
}
@@ -170,7 +174,7 @@
}
webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats() const {
- RTC_DCHECK_RUN_ON(&thread_checker_);
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
webrtc::AudioReceiveStream::Stats stats;
stats.remote_ssrc = config_.rtp.remote_ssrc;
ScopedVoEInterface<VoECodec> codec(voice_engine());
@@ -220,22 +224,78 @@
}
void AudioReceiveStream::SetSink(std::unique_ptr<AudioSinkInterface> sink) {
- RTC_DCHECK_RUN_ON(&thread_checker_);
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
channel_proxy_->SetSink(std::move(sink));
}
void AudioReceiveStream::SetGain(float gain) {
- RTC_DCHECK_RUN_ON(&thread_checker_);
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
channel_proxy_->SetChannelOutputVolumeScaling(gain);
}
-const webrtc::AudioReceiveStream::Config& AudioReceiveStream::config() const {
- RTC_DCHECK_RUN_ON(&thread_checker_);
- return config_;
+AudioMixer::Source::AudioFrameInfo AudioReceiveStream::GetAudioFrameWithInfo(
+ int sample_rate_hz,
+ AudioFrame* audio_frame) {
+ return channel_proxy_->GetAudioFrameWithInfo(sample_rate_hz, audio_frame);
+}
+
+int AudioReceiveStream::Ssrc() const {
+ return config_.rtp.remote_ssrc;
+}
+
+int AudioReceiveStream::PreferredSampleRate() const {
+ return channel_proxy_->NeededFrequency();
+}
+
+int AudioReceiveStream::id() const {
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
+ return config_.rtp.remote_ssrc;
+}
+
+rtc::Optional<Syncable::Info> AudioReceiveStream::GetInfo() const {
+ RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
+ Syncable::Info info;
+
+ RtpRtcp* rtp_rtcp = nullptr;
+ RtpReceiver* rtp_receiver = nullptr;
+ channel_proxy_->GetRtpRtcp(&rtp_rtcp, &rtp_receiver);
+ RTC_DCHECK(rtp_rtcp);
+ RTC_DCHECK(rtp_receiver);
+
+ if (!rtp_receiver->Timestamp(&info.latest_received_capture_timestamp)) {
+ return rtc::Optional<Syncable::Info>();
+ }
+ if (!rtp_receiver->LastReceivedTimeMs(&info.latest_receive_time_ms)) {
+ return rtc::Optional<Syncable::Info>();
+ }
+ if (rtp_rtcp->RemoteNTP(&info.capture_time_ntp_secs,
+ &info.capture_time_ntp_frac,
+ nullptr,
+ nullptr,
+ &info.capture_time_source_clock) != 0) {
+ return rtc::Optional<Syncable::Info>();
+ }
+
+ int jitter_buffer_delay_ms = 0;
+ int playout_buffer_delay_ms = 0;
+ channel_proxy_->GetDelayEstimate(&jitter_buffer_delay_ms,
+ &playout_buffer_delay_ms);
+ info.current_delay_ms = jitter_buffer_delay_ms + playout_buffer_delay_ms;
+ return rtc::Optional<Syncable::Info>(info);
+}
+
+uint32_t AudioReceiveStream::GetPlayoutTimestamp() const {
+ // Called on video capture thread.
+ return channel_proxy_->GetPlayoutTimestamp();
+}
+
+void AudioReceiveStream::SetMinimumPlayoutDelay(int delay_ms) {
+ RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
+ return channel_proxy_->SetMinimumPlayoutDelay(delay_ms);
}
void AudioReceiveStream::AssociateSendStream(AudioSendStream* send_stream) {
- RTC_DCHECK(thread_checker_.CalledOnValidThread());
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
if (send_stream) {
VoiceEngineImpl* voe_impl = static_cast<VoiceEngineImpl*>(voice_engine());
std::unique_ptr<voe::ChannelProxy> send_channel_proxy =
@@ -247,7 +307,7 @@
}
void AudioReceiveStream::SignalNetworkState(NetworkState state) {
- RTC_DCHECK_RUN_ON(&thread_checker_);
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
}
bool AudioReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) {
@@ -286,24 +346,9 @@
return channel_proxy_->ReceivedRTPPacket(packet, length, packet_time);
}
-AudioMixer::Source::AudioFrameInfo AudioReceiveStream::GetAudioFrameWithInfo(
- int sample_rate_hz,
- AudioFrame* audio_frame) {
- return channel_proxy_->GetAudioFrameWithInfo(sample_rate_hz, audio_frame);
-}
-
-int AudioReceiveStream::PreferredSampleRate() const {
- return channel_proxy_->NeededFrequency();
-}
-
-int AudioReceiveStream::Ssrc() const {
- return config_.rtp.remote_ssrc;
-}
-
-internal::AudioState* AudioReceiveStream::audio_state() const {
- auto* audio_state = static_cast<internal::AudioState*>(audio_state_.get());
- RTC_DCHECK(audio_state);
- return audio_state;
+const webrtc::AudioReceiveStream::Config& AudioReceiveStream::config() const {
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
+ return config_;
}
VoiceEngine* AudioReceiveStream::voice_engine() const {
@@ -312,6 +357,12 @@
return voice_engine;
}
+internal::AudioState* AudioReceiveStream::audio_state() const {
+ auto* audio_state = static_cast<internal::AudioState*>(audio_state_.get());
+ RTC_DCHECK(audio_state);
+ return audio_state;
+}
+
int AudioReceiveStream::SetVoiceEnginePlayout(bool playout) {
ScopedVoEInterface<VoEBase> base(voice_engine());
if (playout) {
@@ -320,6 +371,5 @@
return base->StopPlayout(config_.voe_channel_id);
}
}
-
} // namespace internal
} // namespace webrtc
diff --git a/webrtc/audio/audio_receive_stream.h b/webrtc/audio/audio_receive_stream.h
index 7dfc5d6..6721c7e 100644
--- a/webrtc/audio/audio_receive_stream.h
+++ b/webrtc/audio/audio_receive_stream.h
@@ -18,13 +18,13 @@
#include "webrtc/base/constructormagic.h"
#include "webrtc/base/thread_checker.h"
#include "webrtc/call/audio_receive_stream.h"
-#include "webrtc/call/audio_state.h"
+#include "webrtc/call/syncable.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
namespace webrtc {
+class PacketRouter;
class RemoteBitrateEstimator;
class RtcEventLog;
-class PacketRouter;
namespace voe {
class ChannelProxy;
@@ -34,7 +34,8 @@
class AudioSendStream;
class AudioReceiveStream final : public webrtc::AudioReceiveStream,
- public AudioMixer::Source {
+ public AudioMixer::Source,
+ public Syncable {
public:
AudioReceiveStream(PacketRouter* packet_router,
RemoteBitrateEstimator* remote_bitrate_estimator,
@@ -50,6 +51,18 @@
void SetSink(std::unique_ptr<AudioSinkInterface> sink) override;
void SetGain(float gain) override;
+ // AudioMixer::Source
+ AudioFrameInfo GetAudioFrameWithInfo(int sample_rate_hz,
+ AudioFrame* audio_frame) override;
+ int Ssrc() const override;
+ int PreferredSampleRate() const override;
+
+ // Syncable
+ int id() const override;
+ rtc::Optional<Syncable::Info> GetInfo() const override;
+ uint32_t GetPlayoutTimestamp() const override;
+ void SetMinimumPlayoutDelay(int delay_ms) override;
+
void AssociateSendStream(AudioSendStream* send_stream);
void SignalNetworkState(NetworkState state);
bool DeliverRtcp(const uint8_t* packet, size_t length);
@@ -58,25 +71,20 @@
const PacketTime& packet_time);
const webrtc::AudioReceiveStream::Config& config() const;
- // AudioMixer::Source
- AudioFrameInfo GetAudioFrameWithInfo(int sample_rate_hz,
- AudioFrame* audio_frame) override;
- int PreferredSampleRate() const override;
- int Ssrc() const override;
-
private:
VoiceEngine* voice_engine() const;
AudioState* audio_state() const;
int SetVoiceEnginePlayout(bool playout);
- rtc::ThreadChecker thread_checker_;
+ rtc::ThreadChecker worker_thread_checker_;
+ rtc::ThreadChecker module_process_thread_checker_;
RemoteBitrateEstimator* const remote_bitrate_estimator_;
const webrtc::AudioReceiveStream::Config config_;
rtc::scoped_refptr<webrtc::AudioState> audio_state_;
std::unique_ptr<RtpHeaderParser> rtp_header_parser_;
std::unique_ptr<voe::ChannelProxy> channel_proxy_;
- bool playing_ ACCESS_ON(thread_checker_) = false;
+ bool playing_ ACCESS_ON(worker_thread_checker_) = false;
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioReceiveStream);
};
diff --git a/webrtc/call/BUILD.gn b/webrtc/call/BUILD.gn
index acad72d..382ae2e 100644
--- a/webrtc/call/BUILD.gn
+++ b/webrtc/call/BUILD.gn
@@ -16,6 +16,8 @@
"audio_state.h",
"call.h",
"flexfec_receive_stream.h",
+ "syncable.cc",
+ "syncable.h",
]
}
diff --git a/webrtc/call/call.cc b/webrtc/call/call.cc
index 88faf87..6aa564e 100644
--- a/webrtc/call/call.cc
+++ b/webrtc/call/call.cc
@@ -55,7 +55,6 @@
#include "webrtc/video/video_receive_stream.h"
#include "webrtc/video/video_send_stream.h"
#include "webrtc/video/vie_remb.h"
-#include "webrtc/voice_engine/include/voe_codec.h"
namespace webrtc {
@@ -146,15 +145,6 @@
void ConfigureSync(const std::string& sync_group)
EXCLUSIVE_LOCKS_REQUIRED(receive_crit_);
- VoiceEngine* voice_engine() {
- internal::AudioState* audio_state =
- static_cast<internal::AudioState*>(config_.audio_state.get());
- if (audio_state)
- return audio_state->voice_engine();
- else
- return nullptr;
- }
-
rtc::Optional<RtpPacketReceived> ParseRtpPacket(const uint8_t* packet,
size_t length,
const PacketTime& packet_time)
@@ -648,8 +638,8 @@
}
VideoReceiveStream* receive_stream = new VideoReceiveStream(
num_cpu_cores_, protected_by_flexfec, congestion_controller_.get(),
- &packet_router_, std::move(configuration), voice_engine(),
- module_process_thread_.get(), call_stats_.get(), &remb_);
+ &packet_router_, std::move(configuration), module_process_thread_.get(),
+ call_stats_.get(), &remb_);
const webrtc::VideoReceiveStream::Config& config = receive_stream->config();
{
@@ -1019,7 +1009,7 @@
void Call::ConfigureSync(const std::string& sync_group) {
// Set sync only if there was no previous one.
- if (voice_engine() == nullptr || sync_group.empty())
+ if (sync_group.empty())
return;
AudioReceiveStream* sync_audio_stream = nullptr;
@@ -1056,11 +1046,11 @@
"the current implementation.";
}
// Only sync the first A/V pair within this sync group.
- if (sync_audio_stream != nullptr && num_synced_streams == 1) {
- video_stream->SetSyncChannel(voice_engine(),
- sync_audio_stream->config().voe_channel_id);
+ if (num_synced_streams == 1) {
+ // sync_audio_stream may be null and that's ok.
+ video_stream->SetSync(sync_audio_stream);
} else {
- video_stream->SetSyncChannel(voice_engine(), -1);
+ video_stream->SetSync(nullptr);
}
}
}
diff --git a/webrtc/call/syncable.cc b/webrtc/call/syncable.cc
new file mode 100644
index 0000000..c254e4f
--- /dev/null
+++ b/webrtc/call/syncable.cc
@@ -0,0 +1,17 @@
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/call/syncable.h"
+
+namespace webrtc {
+
+Syncable::~Syncable() = default;
+
+} // namespace webrtc
diff --git a/webrtc/call/syncable.h b/webrtc/call/syncable.h
new file mode 100644
index 0000000..3459f9e
--- /dev/null
+++ b/webrtc/call/syncable.h
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+// Syncable is used by RtpStreamsSynchronizer in VideoReceiveStream, and
+// implemented by AudioReceiveStream.
+
+#ifndef WEBRTC_CALL_SYNCABLE_H_
+#define WEBRTC_CALL_SYNCABLE_H_
+
+#include <stdint.h>
+
+#include "webrtc/base/optional.h"
+
+namespace webrtc {
+
+class Syncable {
+ public:
+ struct Info {
+ int64_t latest_receive_time_ms = 0;
+ uint32_t latest_received_capture_timestamp = 0;
+ uint32_t capture_time_ntp_secs = 0;
+ uint32_t capture_time_ntp_frac = 0;
+ uint32_t capture_time_source_clock = 0;
+ int current_delay_ms = 0;
+ };
+
+ virtual ~Syncable();
+
+ virtual int id() const = 0;
+ virtual rtc::Optional<Info> GetInfo() const = 0;
+ virtual uint32_t GetPlayoutTimestamp() const = 0;
+ virtual void SetMinimumPlayoutDelay(int delay_ms) = 0;
+};
+} // namespace webrtc
+
+#endif // WEBRTC_CALL_SYNCABLE_H_
diff --git a/webrtc/test/mock_voe_channel_proxy.h b/webrtc/test/mock_voe_channel_proxy.h
index 44f3c7c..dc2c532 100644
--- a/webrtc/test/mock_voe_channel_proxy.h
+++ b/webrtc/test/mock_voe_channel_proxy.h
@@ -73,6 +73,12 @@
MOCK_METHOD1(AssociateSendChannel,
void(const ChannelProxy& send_channel_proxy));
MOCK_METHOD0(DisassociateSendChannel, void());
+ MOCK_CONST_METHOD2(GetRtpRtcp, void(RtpRtcp** rtp_rtcp,
+ RtpReceiver** rtp_receiver));
+ MOCK_CONST_METHOD2(GetDelayEstimate, void(int* jitter_buffer_delay_ms,
+ int* playout_buffer_delay_ms));
+ MOCK_CONST_METHOD0(GetPlayoutTimestamp, uint32_t());
+ MOCK_METHOD1(SetMinimumPlayoutDelay, void(int delay_ms));
};
} // namespace test
} // namespace webrtc
diff --git a/webrtc/video/rtp_streams_synchronizer.cc b/webrtc/video/rtp_streams_synchronizer.cc
index 0d026b3..1edb9b8 100644
--- a/webrtc/video/rtp_streams_synchronizer.cc
+++ b/webrtc/video/rtp_streams_synchronizer.cc
@@ -14,83 +14,48 @@
#include "webrtc/base/logging.h"
#include "webrtc/base/timeutils.h"
#include "webrtc/base/trace_event.h"
-#include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h"
-#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
+#include "webrtc/call/syncable.h"
#include "webrtc/modules/video_coding/video_coding_impl.h"
-#include "webrtc/system_wrappers/include/clock.h"
-#include "webrtc/video/stream_synchronization.h"
-#include "webrtc/video_frame.h"
-#include "webrtc/voice_engine/include/voe_video_sync.h"
namespace webrtc {
namespace {
bool UpdateMeasurements(StreamSynchronization::Measurements* stream,
- RtpRtcp* rtp_rtcp,
- RtpReceiver* receiver) {
- if (!receiver->Timestamp(&stream->latest_timestamp))
- return false;
- if (!receiver->LastReceivedTimeMs(&stream->latest_receive_time_ms))
- return false;
-
- uint32_t ntp_secs = 0;
- uint32_t ntp_frac = 0;
- uint32_t rtp_timestamp = 0;
- if (rtp_rtcp->RemoteNTP(&ntp_secs, &ntp_frac, nullptr, nullptr,
- &rtp_timestamp) != 0) {
- return false;
- }
-
+ const Syncable::Info& info) {
+ RTC_DCHECK(stream);
+ stream->latest_timestamp = info.latest_received_capture_timestamp;
+ stream->latest_receive_time_ms = info.latest_receive_time_ms;
bool new_rtcp_sr = false;
- if (!stream->rtp_to_ntp.UpdateMeasurements(ntp_secs, ntp_frac, rtp_timestamp,
+ if (!stream->rtp_to_ntp.UpdateMeasurements(info.capture_time_ntp_secs,
+ info.capture_time_ntp_frac,
+ info.capture_time_source_clock,
&new_rtcp_sr)) {
return false;
}
-
return true;
}
} // namespace
-RtpStreamsSynchronizer::RtpStreamsSynchronizer(
- vcm::VideoReceiver* video_receiver,
- RtpStreamReceiver* rtp_stream_receiver)
- : clock_(Clock::GetRealTimeClock()),
- video_receiver_(video_receiver),
- video_rtp_receiver_(rtp_stream_receiver->GetRtpReceiver()),
- video_rtp_rtcp_(rtp_stream_receiver->rtp_rtcp()),
- voe_channel_id_(-1),
- voe_sync_interface_(nullptr),
- audio_rtp_receiver_(nullptr),
- audio_rtp_rtcp_(nullptr),
+RtpStreamsSynchronizer::RtpStreamsSynchronizer(Syncable* syncable_video)
+ : syncable_video_(syncable_video),
+ syncable_audio_(nullptr),
sync_(),
last_sync_time_(rtc::TimeNanos()) {
+ RTC_DCHECK(syncable_video);
process_thread_checker_.DetachFromThread();
}
-void RtpStreamsSynchronizer::ConfigureSync(int voe_channel_id,
- VoEVideoSync* voe_sync_interface) {
- if (voe_channel_id != -1)
- RTC_DCHECK(voe_sync_interface);
-
+void RtpStreamsSynchronizer::ConfigureSync(Syncable* syncable_audio) {
rtc::CritScope lock(&crit_);
- if (voe_channel_id_ == voe_channel_id &&
- voe_sync_interface_ == voe_sync_interface) {
+ if (syncable_audio == syncable_audio_) {
// This prevents expensive no-ops.
return;
}
- voe_channel_id_ = voe_channel_id;
- voe_sync_interface_ = voe_sync_interface;
- audio_rtp_rtcp_ = nullptr;
- audio_rtp_receiver_ = nullptr;
+ syncable_audio_ = syncable_audio;
sync_.reset(nullptr);
-
- if (voe_channel_id_ != -1) {
- voe_sync_interface_->GetRtpRtcp(voe_channel_id_, &audio_rtp_rtcp_,
- &audio_rtp_receiver_);
- RTC_DCHECK(audio_rtp_rtcp_);
- RTC_DCHECK(audio_rtp_receiver_);
- sync_.reset(new StreamSynchronization(video_rtp_rtcp_->SSRC(),
- voe_channel_id_));
+ if (syncable_audio_) {
+ sync_.reset(new StreamSynchronization(syncable_video_->id(),
+ syncable_audio_->id()));
}
}
@@ -103,35 +68,22 @@
void RtpStreamsSynchronizer::Process() {
RTC_DCHECK_RUN_ON(&process_thread_checker_);
-
- const int current_video_delay_ms = video_receiver_->Delay();
last_sync_time_ = rtc::TimeNanos();
rtc::CritScope lock(&crit_);
- if (voe_channel_id_ == -1) {
+ if (!syncable_audio_) {
return;
}
- RTC_DCHECK(voe_sync_interface_);
RTC_DCHECK(sync_.get());
- int audio_jitter_buffer_delay_ms = 0;
- int playout_buffer_delay_ms = 0;
- if (voe_sync_interface_->GetDelayEstimate(voe_channel_id_,
- &audio_jitter_buffer_delay_ms,
- &playout_buffer_delay_ms) != 0) {
+ rtc::Optional<Syncable::Info> audio_info = syncable_audio_->GetInfo();
+ if (!audio_info || !UpdateMeasurements(&audio_measurement_, *audio_info)) {
return;
}
- const int current_audio_delay_ms = audio_jitter_buffer_delay_ms +
- playout_buffer_delay_ms;
int64_t last_video_receive_ms = video_measurement_.latest_receive_time_ms;
- if (!UpdateMeasurements(&video_measurement_, video_rtp_rtcp_,
- video_rtp_receiver_)) {
- return;
- }
-
- if (!UpdateMeasurements(&audio_measurement_, audio_rtp_rtcp_,
- audio_rtp_receiver_)) {
+ rtc::Optional<Syncable::Info> video_info = syncable_video_->GetInfo();
+ if (!video_info || !UpdateMeasurements(&video_measurement_, *video_info)) {
return;
}
@@ -147,41 +99,38 @@
return;
}
- TRACE_COUNTER1("webrtc", "SyncCurrentVideoDelay", current_video_delay_ms);
- TRACE_COUNTER1("webrtc", "SyncCurrentAudioDelay", current_audio_delay_ms);
+ TRACE_COUNTER1("webrtc", "SyncCurrentVideoDelay",
+ video_info->current_delay_ms);
+ TRACE_COUNTER1("webrtc", "SyncCurrentAudioDelay",
+ audio_info->current_delay_ms);
TRACE_COUNTER1("webrtc", "SyncRelativeDelay", relative_delay_ms);
int target_audio_delay_ms = 0;
- int target_video_delay_ms = current_video_delay_ms;
+ int target_video_delay_ms = video_info->current_delay_ms;
// Calculate the necessary extra audio delay and desired total video
// delay to get the streams in sync.
if (!sync_->ComputeDelays(relative_delay_ms,
- current_audio_delay_ms,
+ audio_info->current_delay_ms,
&target_audio_delay_ms,
&target_video_delay_ms)) {
return;
}
- if (voe_sync_interface_->SetMinimumPlayoutDelay(
- voe_channel_id_, target_audio_delay_ms) == -1) {
- LOG(LS_ERROR) << "Error setting voice delay.";
- }
- video_receiver_->SetMinimumPlayoutDelay(target_video_delay_ms);
+ syncable_audio_->SetMinimumPlayoutDelay(target_audio_delay_ms);
+ syncable_video_->SetMinimumPlayoutDelay(target_video_delay_ms);
}
bool RtpStreamsSynchronizer::GetStreamSyncOffsetInMs(
- const VideoFrame& frame,
+ uint32_t timestamp,
+ int64_t render_time_ms,
int64_t* stream_offset_ms,
double* estimated_freq_khz) const {
rtc::CritScope lock(&crit_);
- if (voe_channel_id_ == -1)
- return false;
-
- uint32_t playout_timestamp = 0;
- if (voe_sync_interface_->GetPlayoutTimestamp(voe_channel_id_,
- playout_timestamp) != 0) {
+ if (!syncable_audio_) {
return false;
}
+ uint32_t playout_timestamp = syncable_audio_->GetPlayoutTimestamp();
+
int64_t latest_audio_ntp;
if (!audio_measurement_.rtp_to_ntp.Estimate(playout_timestamp,
&latest_audio_ntp)) {
@@ -189,13 +138,11 @@
}
int64_t latest_video_ntp;
- if (!video_measurement_.rtp_to_ntp.Estimate(frame.timestamp(),
- &latest_video_ntp)) {
+ if (!video_measurement_.rtp_to_ntp.Estimate(timestamp, &latest_video_ntp)) {
return false;
}
- int64_t time_to_render_ms =
- frame.render_time_ms() - clock_->TimeInMilliseconds();
+ int64_t time_to_render_ms = render_time_ms - rtc::TimeMillis();
if (time_to_render_ms > 0)
latest_video_ntp += time_to_render_ms;
diff --git a/webrtc/video/rtp_streams_synchronizer.h b/webrtc/video/rtp_streams_synchronizer.h
index bc24d6f..afa1501 100644
--- a/webrtc/video/rtp_streams_synchronizer.h
+++ b/webrtc/video/rtp_streams_synchronizer.h
@@ -19,14 +19,11 @@
#include "webrtc/base/criticalsection.h"
#include "webrtc/base/thread_checker.h"
#include "webrtc/modules/include/module.h"
-#include "webrtc/video/rtp_stream_receiver.h"
#include "webrtc/video/stream_synchronization.h"
namespace webrtc {
-class Clock;
-class VideoFrame;
-class VoEVideoSync;
+class Syncable;
namespace vcm {
class VideoReceiver;
@@ -34,11 +31,9 @@
class RtpStreamsSynchronizer : public Module {
public:
- RtpStreamsSynchronizer(vcm::VideoReceiver* vcm,
- RtpStreamReceiver* rtp_stream_receiver);
+ explicit RtpStreamsSynchronizer(Syncable* syncable_video);
- void ConfigureSync(int voe_channel_id,
- VoEVideoSync* voe_sync_interface);
+ void ConfigureSync(Syncable* syncable_audio);
// Implements Module.
int64_t TimeUntilNextProcess() override;
@@ -48,21 +43,16 @@
// video |frame|. Returns true on success, false otherwise.
// The estimated frequency is the frequency used in the RTP to NTP timestamp
// conversion.
- bool GetStreamSyncOffsetInMs(const VideoFrame& frame,
+ bool GetStreamSyncOffsetInMs(uint32_t timestamp,
+ int64_t render_time_ms,
int64_t* stream_offset_ms,
double* estimated_freq_khz) const;
private:
- Clock* const clock_;
- vcm::VideoReceiver* const video_receiver_;
- RtpReceiver* const video_rtp_receiver_;
- RtpRtcp* const video_rtp_rtcp_;
+ Syncable* syncable_video_;
rtc::CriticalSection crit_;
- int voe_channel_id_ GUARDED_BY(crit_);
- VoEVideoSync* voe_sync_interface_ GUARDED_BY(crit_);
- RtpReceiver* audio_rtp_receiver_ GUARDED_BY(crit_);
- RtpRtcp* audio_rtp_rtcp_ GUARDED_BY(crit_);
+ Syncable* syncable_audio_ GUARDED_BY(crit_);
std::unique_ptr<StreamSynchronization> sync_ GUARDED_BY(crit_);
StreamSynchronization::Measurements audio_measurement_ GUARDED_BY(crit_);
StreamSynchronization::Measurements video_measurement_ GUARDED_BY(crit_);
diff --git a/webrtc/video/stream_synchronization.cc b/webrtc/video/stream_synchronization.cc
index 145922c..32a2d72 100644
--- a/webrtc/video/stream_synchronization.cc
+++ b/webrtc/video/stream_synchronization.cc
@@ -26,10 +26,10 @@
// Minimum difference between audio and video to warrant a change.
static const int kMinDeltaMs = 30;
-StreamSynchronization::StreamSynchronization(uint32_t video_primary_ssrc,
- int audio_channel_id)
- : video_primary_ssrc_(video_primary_ssrc),
- audio_channel_id_(audio_channel_id),
+StreamSynchronization::StreamSynchronization(int video_stream_id,
+ int audio_stream_id)
+ : video_stream_id_(video_stream_id),
+ audio_stream_id_(audio_stream_id),
base_target_delay_ms_(0),
avg_diff_ms_(0) {
}
@@ -72,7 +72,7 @@
int current_video_delay_ms = *total_video_delay_target_ms;
LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms
<< " current diff: " << relative_delay_ms
- << " for channel " << audio_channel_id_;
+ << " for stream " << audio_stream_id_;
// Calculate the difference between the lowest possible video delay and
// the current audio delay.
int current_diff_ms = current_video_delay_ms - current_audio_delay_ms +
@@ -166,9 +166,9 @@
channel_delay_.last_audio_delay_ms = new_audio_delay_ms;
LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms
- << " for video primary SSRC " << video_primary_ssrc_
+ << " for video stream " << video_stream_id_
<< " and audio delay " << channel_delay_.extra_audio_delay_ms
- << " for audio channel " << audio_channel_id_;
+ << " for audio stream " << audio_stream_id_;
// Return values.
*total_video_delay_target_ms = new_video_delay_ms;
diff --git a/webrtc/video/stream_synchronization.h b/webrtc/video/stream_synchronization.h
index 0b1ad4f..d32966b 100644
--- a/webrtc/video/stream_synchronization.h
+++ b/webrtc/video/stream_synchronization.h
@@ -27,7 +27,7 @@
uint32_t latest_timestamp;
};
- StreamSynchronization(uint32_t video_primary_ssrc, int audio_channel_id);
+ StreamSynchronization(int video_stream_id, int audio_stream_id);
bool ComputeDelays(int relative_delay_ms,
int current_audio_delay_ms,
@@ -53,8 +53,8 @@
};
SynchronizationDelays channel_delay_;
- const uint32_t video_primary_ssrc_;
- const int audio_channel_id_;
+ const int video_stream_id_;
+ const int audio_stream_id_;
int base_target_delay_ms_;
int avg_diff_ms_;
};
diff --git a/webrtc/video/video_receive_stream.cc b/webrtc/video/video_receive_stream.cc
index 5fd436d..25418c4 100644
--- a/webrtc/video/video_receive_stream.cc
+++ b/webrtc/video/video_receive_stream.cc
@@ -22,6 +22,8 @@
#include "webrtc/common_video/h264/profile_level_id.h"
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
#include "webrtc/modules/congestion_controller/include/congestion_controller.h"
+#include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h"
+#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
#include "webrtc/modules/utility/include/process_thread.h"
#include "webrtc/modules/video_coding/frame_object.h"
#include "webrtc/modules/video_coding/include/video_coding.h"
@@ -33,7 +35,6 @@
#include "webrtc/video/call_stats.h"
#include "webrtc/video/receive_statistics_proxy.h"
#include "webrtc/video_receive_stream.h"
-#include "webrtc/voice_engine/include/voe_video_sync.h"
namespace webrtc {
@@ -190,7 +191,6 @@
CongestionController* congestion_controller,
PacketRouter* packet_router,
VideoReceiveStream::Config config,
- webrtc::VoiceEngine* voice_engine,
ProcessThread* process_thread,
CallStats* call_stats,
VieRemb* remb)
@@ -220,7 +220,7 @@
this, // KeyFrameRequestSender
this, // OnCompleteFrameCallback
timing_.get()),
- rtp_stream_sync_(&video_receiver_, &rtp_stream_receiver_),
+ rtp_stream_sync_(this),
jitter_buffer_experiment_(
field_trial::FindFullName("WebRTC-NewVideoJitterBuffer") ==
"Enabled") {
@@ -230,6 +230,8 @@
RTC_DCHECK(congestion_controller_);
RTC_DCHECK(call_stats_);
+ module_process_thread_checker_.DetachFromThread();
+
RTC_DCHECK(!config_.decoders.empty());
std::set<int> decoder_payload_types;
for (const Decoder& decoder : config_.decoders) {
@@ -254,6 +256,7 @@
}
VideoReceiveStream::~VideoReceiveStream() {
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
LOG(LS_INFO) << "~VideoReceiveStream: " << config_.ToString();
Stop();
@@ -265,6 +268,7 @@
}
void VideoReceiveStream::SignalNetworkState(NetworkState state) {
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
rtp_stream_receiver_.SignalNetworkState(state);
}
@@ -281,21 +285,17 @@
bool VideoReceiveStream::OnRecoveredPacket(const uint8_t* packet,
size_t length) {
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
return rtp_stream_receiver_.OnRecoveredPacket(packet, length);
}
-void VideoReceiveStream::SetSyncChannel(VoiceEngine* voice_engine,
- int audio_channel_id) {
- if (voice_engine && audio_channel_id != -1) {
- VoEVideoSync* voe_sync_interface = VoEVideoSync::GetInterface(voice_engine);
- rtp_stream_sync_.ConfigureSync(audio_channel_id, voe_sync_interface);
- voe_sync_interface->Release();
- } else {
- rtp_stream_sync_.ConfigureSync(-1, nullptr);
- }
+void VideoReceiveStream::SetSync(Syncable* audio_syncable) {
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
+ rtp_stream_sync_.ConfigureSync(audio_syncable);
}
void VideoReceiveStream::Start() {
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
if (decode_thread_.IsRunning())
return;
@@ -346,6 +346,7 @@
}
void VideoReceiveStream::Stop() {
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
rtp_stream_receiver_.StopReceive();
// TriggerDecoderShutdown will release any waiting decoder thread and make it
// stop immediately, instead of waiting for a timeout. Needs to be called
@@ -407,7 +408,9 @@
// function itself, another in GetChannel() and a third in
// GetPlayoutTimestamp. Seems excessive. Anyhow, I'm assuming the function
// succeeds most of the time, which leads to grabbing a fourth lock.
- if (rtp_stream_sync_.GetStreamSyncOffsetInMs(video_frame, &sync_offset_ms,
+ if (rtp_stream_sync_.GetStreamSyncOffsetInMs(video_frame.timestamp(),
+ video_frame.render_time_ms(),
+ &sync_offset_ms,
&estimated_freq_khz)) {
// TODO(tommi): OnSyncOffsetUpdated grabs a lock.
stats_proxy_.OnSyncOffsetUpdated(sync_offset_ms, estimated_freq_khz);
@@ -461,6 +464,46 @@
rtp_stream_receiver_.FrameContinuous(last_continuous_pid);
}
+int VideoReceiveStream::id() const {
+ RTC_DCHECK_RUN_ON(&worker_thread_checker_);
+ return config_.rtp.remote_ssrc;
+}
+
+rtc::Optional<Syncable::Info> VideoReceiveStream::GetInfo() const {
+ RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
+ Syncable::Info info;
+
+ RtpReceiver* rtp_receiver = rtp_stream_receiver_.GetRtpReceiver();
+ RTC_DCHECK(rtp_receiver);
+ if (!rtp_receiver->Timestamp(&info.latest_received_capture_timestamp))
+ return rtc::Optional<Syncable::Info>();
+ if (!rtp_receiver->LastReceivedTimeMs(&info.latest_receive_time_ms))
+ return rtc::Optional<Syncable::Info>();
+
+ RtpRtcp* rtp_rtcp = rtp_stream_receiver_.rtp_rtcp();
+ RTC_DCHECK(rtp_rtcp);
+ if (rtp_rtcp->RemoteNTP(&info.capture_time_ntp_secs,
+ &info.capture_time_ntp_frac,
+ nullptr,
+ nullptr,
+ &info.capture_time_source_clock) != 0) {
+ return rtc::Optional<Syncable::Info>();
+ }
+
+ info.current_delay_ms = video_receiver_.Delay();
+ return rtc::Optional<Syncable::Info>(info);
+}
+
+uint32_t VideoReceiveStream::GetPlayoutTimestamp() const {
+ RTC_NOTREACHED();
+ return 0;
+}
+
+void VideoReceiveStream::SetMinimumPlayoutDelay(int delay_ms) {
+ RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
+ video_receiver_.SetMinimumPlayoutDelay(delay_ms);
+}
+
bool VideoReceiveStream::DecodeThreadFunction(void* ptr) {
static_cast<VideoReceiveStream*>(ptr)->Decode();
return true;
@@ -489,6 +532,5 @@
video_receiver_.Decode(kMaxDecodeWaitTimeMs);
}
}
-
} // namespace internal
} // namespace webrtc
diff --git a/webrtc/video/video_receive_stream.h b/webrtc/video/video_receive_stream.h
index 9960dc9..063a4e4 100644
--- a/webrtc/video/video_receive_stream.h
+++ b/webrtc/video/video_receive_stream.h
@@ -14,6 +14,8 @@
#include <memory>
#include <vector>
+#include "webrtc/base/thread_checker.h"
+#include "webrtc/call/syncable.h"
#include "webrtc/common_video/include/incoming_video_stream.h"
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
#include "webrtc/modules/rtp_rtcp/include/flexfec_receiver.h"
@@ -34,7 +36,6 @@
class IvfFileWriter;
class ProcessThread;
class RTPFragmentationHeader;
-class VoiceEngine;
class VieRemb;
class VCMTiming;
class VCMJitterEstimator;
@@ -46,14 +47,14 @@
public EncodedImageCallback,
public NackSender,
public KeyFrameRequestSender,
- public video_coding::OnCompleteFrameCallback {
+ public video_coding::OnCompleteFrameCallback,
+ public Syncable {
public:
VideoReceiveStream(int num_cpu_cores,
bool protected_by_flexfec,
CongestionController* congestion_controller,
PacketRouter* packet_router,
VideoReceiveStream::Config config,
- webrtc::VoiceEngine* voice_engine,
ProcessThread* process_thread,
CallStats* call_stats,
VieRemb* remb);
@@ -69,7 +70,7 @@
bool OnRecoveredPacket(const uint8_t* packet, size_t length);
- void SetSyncChannel(VoiceEngine* voice_engine, int audio_channel_id);
+ void SetSync(Syncable* audio_syncable);
// Implements webrtc::VideoReceiveStream.
void Start() override;
@@ -104,10 +105,19 @@
void OnCompleteFrame(
std::unique_ptr<video_coding::FrameObject> frame) override;
+ // Implements Syncable.
+ int id() const override;
+ rtc::Optional<Syncable::Info> GetInfo() const override;
+ uint32_t GetPlayoutTimestamp() const override;
+ void SetMinimumPlayoutDelay(int delay_ms) override;
+
private:
static bool DecodeThreadFunction(void* ptr);
void Decode();
+ rtc::ThreadChecker worker_thread_checker_;
+ rtc::ThreadChecker module_process_thread_checker_;
+
TransportAdapter transport_adapter_;
const VideoReceiveStream::Config config_;
const int num_cpu_cores_;
diff --git a/webrtc/voice_engine/channel_proxy.cc b/webrtc/voice_engine/channel_proxy.cc
index 59b920f..b3c3a98 100644
--- a/webrtc/voice_engine/channel_proxy.cc
+++ b/webrtc/voice_engine/channel_proxy.cc
@@ -256,6 +256,42 @@
channel()->set_associate_send_channel(ChannelOwner(nullptr));
}
+void ChannelProxy::GetRtpRtcp(RtpRtcp** rtp_rtcp,
+ RtpReceiver** rtp_receiver) const {
+ // Called on Call's module_process_thread_.
+ RTC_DCHECK(rtp_rtcp);
+ RTC_DCHECK(rtp_receiver);
+ int error = channel()->GetRtpRtcp(rtp_rtcp, rtp_receiver);
+ RTC_DCHECK_EQ(0, error);
+}
+
+void ChannelProxy::GetDelayEstimate(int* jitter_buffer_delay_ms,
+ int* playout_buffer_delay_ms) const {
+ // Called on Call's module_process_thread_.
+ RTC_DCHECK(jitter_buffer_delay_ms);
+ RTC_DCHECK(playout_buffer_delay_ms);
+ bool error = channel()->GetDelayEstimate(jitter_buffer_delay_ms,
+ playout_buffer_delay_ms);
+ RTC_DCHECK(error);
+}
+
+uint32_t ChannelProxy::GetPlayoutTimestamp() const {
+ // Called on video capture thread.
+ unsigned int timestamp = 0;
+ int error = channel()->GetPlayoutTimestamp(timestamp);
+ RTC_DCHECK(!error || timestamp == 0);
+ return timestamp;
+}
+
+void ChannelProxy::SetMinimumPlayoutDelay(int delay_ms) {
+ // Called on Call's module_process_thread_.
+ // Limit to range accepted by both VoE and ACM, so we're at least getting as
+ // close as possible, instead of failing.
+ delay_ms = std::max(0, std::min(delay_ms, 10000));
+ int error = channel()->SetMinimumPlayoutDelay(delay_ms);
+ RTC_DCHECK_EQ(0, error);
+}
+
void ChannelProxy::SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats) {
RTC_DCHECK(thread_checker_.CalledOnValidThread());
channel()->SetRtcpRttStats(rtcp_rtt_stats);
diff --git a/webrtc/voice_engine/channel_proxy.h b/webrtc/voice_engine/channel_proxy.h
index 4bd76a4..7d18a74 100644
--- a/webrtc/voice_engine/channel_proxy.h
+++ b/webrtc/voice_engine/channel_proxy.h
@@ -29,6 +29,8 @@
class RtcEventLog;
class RtcpRttStats;
class RtpPacketSender;
+class RtpReceiver;
+class RtpRtcp;
class Transport;
class TransportFeedbackObserver;
@@ -99,6 +101,12 @@
virtual void SetTransportOverhead(int transport_overhead_per_packet);
virtual void AssociateSendChannel(const ChannelProxy& send_channel_proxy);
virtual void DisassociateSendChannel();
+ virtual void GetRtpRtcp(RtpRtcp** rtp_rtcp,
+ RtpReceiver** rtp_receiver) const;
+ virtual void GetDelayEstimate(int* jitter_buffer_delay_ms,
+ int* playout_buffer_delay_ms) const;
+ virtual uint32_t GetPlayoutTimestamp() const;
+ virtual void SetMinimumPlayoutDelay(int delay_ms);
virtual void SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats);