Transform encoded frames in ChannelSend.
This change is part of the implementation of the Insertable Streams Web
API: https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md
Design doc for WebRTC library changes:
http://doc/1eiLkjNUkRy2FssCPLUp6eH08BZuXXoHfbbBP1ZN7EVk
Bug: webrtc:11380
Change-Id: I75444283ddb7f8db742687b497bf532c6dda47be
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171871
Commit-Queue: Marina Ciocea <marinaciocea@webrtc.org>
Reviewed-by: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30952}
diff --git a/audio/BUILD.gn b/audio/BUILD.gn
index 7da3c79..cc52664 100644
--- a/audio/BUILD.gn
+++ b/audio/BUILD.gn
@@ -28,6 +28,8 @@
"channel_receive.h",
"channel_send.cc",
"channel_send.h",
+ "channel_send_frame_transformer_delegate.cc",
+ "channel_send_frame_transformer_delegate.h",
"conversion.h",
"null_audio_poller.cc",
"null_audio_poller.h",
diff --git a/audio/channel_send.cc b/audio/channel_send.cc
index d8ac39c..21f36aa 100644
--- a/audio/channel_send.cc
+++ b/audio/channel_send.cc
@@ -21,6 +21,7 @@
#include "api/call/transport.h"
#include "api/crypto/frame_encryptor_interface.h"
#include "api/rtc_event_log/rtc_event_log.h"
+#include "audio/channel_send_frame_transformer_delegate.h"
#include "audio/utility/audio_frame_operations.h"
#include "call/rtp_transport_controller_send_interface.h"
#include "logging/rtc_event_log/events/rtc_event_audio_playout.h"
@@ -170,6 +171,9 @@
void OnReceivedRtt(int64_t rtt_ms);
+ void InitFrameTransformerDelegate(
+ rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer);
+
// Thread checkers document and lock usage of some methods on voe::Channel to
// specific threads we know about. The goal is to eventually split up
// voe::Channel into parts with single-threaded semantics, and thereby reduce
@@ -224,9 +228,11 @@
// E2EE Frame Encryption Options
const webrtc::CryptoOptions crypto_options_;
- // Frame transformer used by insertable streams to transform encoded frames.
- rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_
- RTC_GUARDED_BY(encoder_queue_);
+ // Delegates calls to a frame transformer to transform audio, and
+ // receives callbacks with the transformed frames; delegates calls to
+ // ChannelSend::SendRtpAudio to send the transformed audio.
+ rtc::scoped_refptr<ChannelSendFrameTransformerDelegate>
+ frame_transformer_delegate_ RTC_GUARDED_BY(encoder_queue_);
rtc::CriticalSection bitrate_crit_section_;
int configured_bitrate_bps_ RTC_GUARDED_BY(bitrate_crit_section_) = 0;
@@ -379,6 +385,14 @@
int64_t absolute_capture_timestamp_ms) {
RTC_DCHECK_RUN_ON(&encoder_queue_);
rtc::ArrayView<const uint8_t> payload(payloadData, payloadSize);
+ if (frame_transformer_delegate_) {
+ // Asynchronously transform the payload before sending it. After the payload
+ // is transformed, the delegate will call SendRtpAudio to send it.
+ frame_transformer_delegate_->Transform(
+ frameType, payloadType, rtp_timestamp, payloadData, payloadSize,
+ absolute_capture_timestamp_ms, _rtpRtcpModule->SSRC());
+ return 0;
+ }
return SendRtpAudio(frameType, payloadType, rtp_timestamp, payload,
absolute_capture_timestamp_ms);
}
@@ -491,7 +505,6 @@
new RateLimiter(clock, kMaxRetransmissionWindowMs)),
frame_encryptor_(frame_encryptor),
crypto_options_(crypto_options),
- frame_transformer_(std::move(frame_transformer)),
encoder_queue_(task_queue_factory->CreateTaskQueue(
"AudioEncoder",
TaskQueueFactory::Priority::NORMAL)) {
@@ -532,11 +545,17 @@
int error = audio_coding_->RegisterTransportCallback(this);
RTC_DCHECK_EQ(0, error);
+ if (frame_transformer)
+ InitFrameTransformerDelegate(std::move(frame_transformer));
}
ChannelSend::~ChannelSend() {
RTC_DCHECK(construction_thread_.IsCurrent());
+ // Resets the delegate's callback to ChannelSend::SendRtpAudio.
+ if (frame_transformer_delegate_)
+ frame_transformer_delegate_->Reset();
+
StopSend();
int error = audio_coding_->RegisterTransportCallback(NULL);
RTC_DCHECK_EQ(0, error);
@@ -915,10 +934,13 @@
void ChannelSend::SetEncoderToPacketizerFrameTransformer(
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
+ if (!frame_transformer)
+ return;
+
encoder_queue_.PostTask(
[this, frame_transformer = std::move(frame_transformer)]() mutable {
RTC_DCHECK_RUN_ON(&encoder_queue_);
- frame_transformer_ = std::move(frame_transformer);
+ InitFrameTransformerDelegate(std::move(frame_transformer));
});
}
@@ -928,6 +950,29 @@
[rtt_ms](AudioEncoder* encoder) { encoder->OnReceivedRtt(rtt_ms); });
}
+void ChannelSend::InitFrameTransformerDelegate(
+ rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
+ RTC_DCHECK_RUN_ON(&encoder_queue_);
+ RTC_DCHECK(frame_transformer);
+ RTC_DCHECK(!frame_transformer_delegate_);
+
+ // Pass a callback to ChannelSend::SendRtpAudio, to be called by the delegate
+ // to send the transformed audio.
+ ChannelSendFrameTransformerDelegate::SendFrameCallback send_audio_callback =
+ [this](AudioFrameType frameType, uint8_t payloadType,
+ uint32_t rtp_timestamp, rtc::ArrayView<const uint8_t> payload,
+ int64_t absolute_capture_timestamp_ms) {
+ RTC_DCHECK_RUN_ON(&encoder_queue_);
+ return SendRtpAudio(frameType, payloadType, rtp_timestamp, payload,
+ absolute_capture_timestamp_ms);
+ };
+ frame_transformer_delegate_ =
+ new rtc::RefCountedObject<ChannelSendFrameTransformerDelegate>(
+ std::move(send_audio_callback), std::move(frame_transformer),
+ &encoder_queue_);
+ frame_transformer_delegate_->Init();
+}
+
} // namespace
std::unique_ptr<ChannelSendInterface> CreateChannelSend(
diff --git a/audio/channel_send_frame_transformer_delegate.cc b/audio/channel_send_frame_transformer_delegate.cc
new file mode 100644
index 0000000..53df6b0
--- /dev/null
+++ b/audio/channel_send_frame_transformer_delegate.cc
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "audio/channel_send_frame_transformer_delegate.h"
+
+#include <utility>
+
+namespace webrtc {
+namespace {
+
+class TransformableAudioFrame : public TransformableFrameInterface {
+ public:
+ TransformableAudioFrame(AudioFrameType frame_type,
+ uint8_t payload_type,
+ uint32_t rtp_timestamp,
+ const uint8_t* payload_data,
+ size_t payload_size,
+ int64_t absolute_capture_timestamp_ms,
+ uint32_t ssrc)
+ : frame_type_(frame_type),
+ payload_type_(payload_type),
+ rtp_timestamp_(rtp_timestamp),
+ payload_(payload_data, payload_size),
+ absolute_capture_timestamp_ms_(absolute_capture_timestamp_ms),
+ ssrc_(ssrc) {}
+ ~TransformableAudioFrame() override = default;
+ rtc::ArrayView<const uint8_t> GetData() const override { return payload_; }
+ void SetData(rtc::ArrayView<const uint8_t> data) override {
+ payload_.SetData(data.data(), data.size());
+ }
+ uint32_t GetTimestamp() const override { return rtp_timestamp_; }
+ uint32_t GetSsrc() const override { return ssrc_; }
+
+ AudioFrameType GetFrameType() const { return frame_type_; }
+ uint8_t GetPayloadType() const { return payload_type_; }
+ int64_t GetAbsoluteCaptureTimestampMs() const {
+ return absolute_capture_timestamp_ms_;
+ }
+
+ private:
+ AudioFrameType frame_type_;
+ uint8_t payload_type_;
+ uint32_t rtp_timestamp_;
+ rtc::Buffer payload_;
+ int64_t absolute_capture_timestamp_ms_;
+ uint32_t ssrc_;
+};
+} // namespace
+
+ChannelSendFrameTransformerDelegate::ChannelSendFrameTransformerDelegate(
+ SendFrameCallback send_frame_callback,
+ rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
+ rtc::TaskQueue* encoder_queue)
+ : send_frame_callback_(send_frame_callback),
+ frame_transformer_(std::move(frame_transformer)),
+ encoder_queue_(encoder_queue) {}
+
+void ChannelSendFrameTransformerDelegate::Init() {
+ frame_transformer_->RegisterTransformedFrameCallback(
+ rtc::scoped_refptr<TransformedFrameCallback>(this));
+}
+
+void ChannelSendFrameTransformerDelegate::Reset() {
+ frame_transformer_->UnregisterTransformedFrameCallback();
+ frame_transformer_ = nullptr;
+
+ rtc::CritScope lock(&send_lock_);
+ send_frame_callback_ = SendFrameCallback();
+}
+
+void ChannelSendFrameTransformerDelegate::Transform(
+ AudioFrameType frame_type,
+ uint8_t payload_type,
+ uint32_t rtp_timestamp,
+ const uint8_t* payload_data,
+ size_t payload_size,
+ int64_t absolute_capture_timestamp_ms,
+ uint32_t ssrc) {
+ frame_transformer_->Transform(std::make_unique<TransformableAudioFrame>(
+ frame_type, payload_type, rtp_timestamp, payload_data, payload_size,
+ absolute_capture_timestamp_ms, ssrc));
+}
+
+void ChannelSendFrameTransformerDelegate::OnTransformedFrame(
+ std::unique_ptr<TransformableFrameInterface> frame) {
+ rtc::CritScope lock(&send_lock_);
+ if (!send_frame_callback_)
+ return;
+ rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> delegate = this;
+ encoder_queue_->PostTask(
+ [delegate = std::move(delegate), frame = std::move(frame)]() mutable {
+ delegate->SendFrame(std::move(frame));
+ });
+}
+
+void ChannelSendFrameTransformerDelegate::SendFrame(
+ std::unique_ptr<TransformableFrameInterface> frame) const {
+ rtc::CritScope lock(&send_lock_);
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ if (!send_frame_callback_)
+ return;
+ auto* transformed_frame = static_cast<TransformableAudioFrame*>(frame.get());
+ send_frame_callback_(
+ transformed_frame->GetFrameType(), transformed_frame->GetPayloadType(),
+ transformed_frame->GetTimestamp(), transformed_frame->GetData(),
+ transformed_frame->GetAbsoluteCaptureTimestampMs());
+}
+
+} // namespace webrtc
diff --git a/audio/channel_send_frame_transformer_delegate.h b/audio/channel_send_frame_transformer_delegate.h
new file mode 100644
index 0000000..f2655e7
--- /dev/null
+++ b/audio/channel_send_frame_transformer_delegate.h
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef AUDIO_CHANNEL_SEND_FRAME_TRANSFORMER_DELEGATE_H_
+#define AUDIO_CHANNEL_SEND_FRAME_TRANSFORMER_DELEGATE_H_
+
+#include <memory>
+
+#include "api/frame_transformer_interface.h"
+#include "modules/audio_coding/include/audio_coding_module_typedefs.h"
+#include "rtc_base/buffer.h"
+#include "rtc_base/critical_section.h"
+#include "rtc_base/synchronization/sequence_checker.h"
+#include "rtc_base/task_queue.h"
+
+namespace webrtc {
+
+// Delegates calls to FrameTransformerInterface to transform frames, and to
+// ChannelSend to send the transformed frames using |send_frame_callback_| on
+// the |encoder_queue_|.
+// OnTransformedFrame() can be called from any thread, the delegate ensures
+// thread-safe access to the ChannelSend callback.
+class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback {
+ public:
+ using SendFrameCallback =
+ std::function<int32_t(AudioFrameType frameType,
+ uint8_t payloadType,
+ uint32_t rtp_timestamp,
+ rtc::ArrayView<const uint8_t> payload,
+ int64_t absolute_capture_timestamp_ms)>;
+ ChannelSendFrameTransformerDelegate(
+ SendFrameCallback send_frame_callback,
+ rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
+ rtc::TaskQueue* encoder_queue);
+
+ // Registers |this| as callback for |frame_transformer_|, to get the
+ // transformed frames.
+ void Init();
+
+ // Unregisters and releases the |frame_transformer_| reference, and resets
+ // |send_frame_callback_| under lock. Called from ChannelSend destructor to
+ // prevent running the callback on a dangling channel.
+ void Reset();
+
+ // Delegates the call to FrameTransformerInterface::TransformFrame, to
+ // transform the frame asynchronously.
+ void Transform(AudioFrameType frame_type,
+ uint8_t payload_type,
+ uint32_t rtp_timestamp,
+ const uint8_t* payload_data,
+ size_t payload_size,
+ int64_t absolute_capture_timestamp_ms,
+ uint32_t ssrc);
+
+ // Implements TransformedFrameCallback. Can be called on any thread.
+ void OnTransformedFrame(
+ std::unique_ptr<TransformableFrameInterface> frame) override;
+
+ // Delegates the call to ChannelSend::SendRtpAudio on the |encoder_queue_|,
+ // by calling |send_audio_callback_|.
+ void SendFrame(std::unique_ptr<TransformableFrameInterface> frame) const;
+
+ protected:
+ ~ChannelSendFrameTransformerDelegate() override = default;
+
+ private:
+ rtc::CriticalSection send_lock_;
+ SendFrameCallback send_frame_callback_ RTC_GUARDED_BY(send_lock_);
+ rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_;
+ rtc::TaskQueue* encoder_queue_ RTC_GUARDED_BY(send_lock_);
+};
+} // namespace webrtc
+#endif // AUDIO_CHANNEL_SEND_FRAME_TRANSFORMER_DELEGATE_H_