Reland "Reland "Refactors UlpFec and FlexFec to use a common interface.""
This is a reland of 49734dc0faa69616a58a1a95c7fc61a4610793cf
Patchset 2 contains a fix for the fuzzer set up. Since we now parse
an RtpPacket out of the fuzzer data, the header needs to be correct,
otherwise we fail before even reaching the FEC code that we actually
want to test.
Bug: webrtc:11340, chromium:1052323, chromium:1055974
TBR=stefan@webrtc.org
Original change's description:
> Reland "Refactors UlpFec and FlexFec to use a common interface."
>
> This is a reland of 11af1d7444fd7438766b7bc52cbd64752d72e32e
>
> Original change's description:
> > Refactors UlpFec and FlexFec to use a common interface.
> >
> > The new VideoFecGenerator is now injected into RtpSenderVideo,
> > and generalizes the usage.
> > This also prepares for being able to genera FEC in the RTP egress
> > module.
> >
> > Bug: webrtc:11340
> > Change-Id: I8aa873129b2fb4131eb3399ee88f6ea2747155a3
> > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/168347
> > Reviewed-by: Stefan Holmer <stefan@webrtc.org>
> > Reviewed-by: Sebastian Jansson <srte@webrtc.org>
> > Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
> > Commit-Queue: Erik Språng <sprang@webrtc.org>
> > Cr-Commit-Position: refs/heads/master@{#30515}
>
> Bug: webrtc:11340, chromium:1052323
> Change-Id: Id646047365f1c46cca9e6f3e8eefa5151207b4a0
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/168608
> Commit-Queue: Erik Språng <sprang@webrtc.org>
> Reviewed-by: Stefan Holmer <stefan@webrtc.org>
> Cr-Commit-Position: refs/heads/master@{#30593}
Bug: webrtc:11340, chromium:1052323
Change-Id: Ib8925f44e2edfcfeadc95c845c3bfc23822604ed
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169222
Commit-Queue: Erik Språng <sprang@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30724}
diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc
index 9eb789c..fba646e 100644
--- a/call/rtp_video_sender.cc
+++ b/call/rtp_video_sender.cc
@@ -36,9 +36,13 @@
namespace webrtc_internal_rtp_video_sender {
-RtpStreamSender::RtpStreamSender(std::unique_ptr<RtpRtcp> rtp_rtcp,
- std::unique_ptr<RTPSenderVideo> sender_video)
- : rtp_rtcp(std::move(rtp_rtcp)), sender_video(std::move(sender_video)) {}
+RtpStreamSender::RtpStreamSender(
+ std::unique_ptr<RtpRtcp> rtp_rtcp,
+ std::unique_ptr<RTPSenderVideo> sender_video,
+ std::unique_ptr<VideoFecGenerator> fec_generator)
+ : rtp_rtcp(std::move(rtp_rtcp)),
+ sender_video(std::move(sender_video)),
+ fec_generator(std::move(fec_generator)) {}
RtpStreamSender::~RtpStreamSender() = default;
@@ -113,6 +117,67 @@
return should_disable_red_and_ulpfec;
}
+// TODO(brandtr): Update this function when we support multistream protection.
+std::unique_ptr<VideoFecGenerator> MaybeCreateFecGenerator(
+ Clock* clock,
+ const RtpConfig& rtp,
+ const std::map<uint32_t, RtpState>& suspended_ssrcs,
+ int simulcast_index) {
+ // If flexfec is configured that takes priority.
+ if (rtp.flexfec.payload_type >= 0) {
+ RTC_DCHECK_GE(rtp.flexfec.payload_type, 0);
+ RTC_DCHECK_LE(rtp.flexfec.payload_type, 127);
+ if (rtp.flexfec.ssrc == 0) {
+ RTC_LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. "
+ "Therefore disabling FlexFEC.";
+ return nullptr;
+ }
+ if (rtp.flexfec.protected_media_ssrcs.empty()) {
+ RTC_LOG(LS_WARNING)
+ << "FlexFEC is enabled, but no protected media SSRC given. "
+ "Therefore disabling FlexFEC.";
+ return nullptr;
+ }
+
+ if (rtp.flexfec.protected_media_ssrcs.size() > 1) {
+ RTC_LOG(LS_WARNING)
+ << "The supplied FlexfecConfig contained multiple protected "
+ "media streams, but our implementation currently only "
+ "supports protecting a single media stream. "
+ "To avoid confusion, disabling FlexFEC completely.";
+ return nullptr;
+ }
+
+ if (absl::c_find(rtp.flexfec.protected_media_ssrcs,
+ rtp.ssrcs[simulcast_index]) ==
+ rtp.flexfec.protected_media_ssrcs.end()) {
+ // Media SSRC not among flexfec protected SSRCs.
+ return nullptr;
+ }
+
+ const RtpState* rtp_state = nullptr;
+ auto it = suspended_ssrcs.find(rtp.flexfec.ssrc);
+ if (it != suspended_ssrcs.end()) {
+ rtp_state = &it->second;
+ }
+
+ RTC_DCHECK_EQ(1U, rtp.flexfec.protected_media_ssrcs.size());
+ return std::make_unique<FlexfecSender>(
+ rtp.flexfec.payload_type, rtp.flexfec.ssrc,
+ rtp.flexfec.protected_media_ssrcs[0], rtp.mid, rtp.extensions,
+ RTPSender::FecExtensionSizes(), rtp_state, clock);
+ } else if (rtp.ulpfec.red_payload_type >= 0 &&
+ rtp.ulpfec.ulpfec_payload_type >= 0 &&
+ !ShouldDisableRedAndUlpfec(/*flexfec_enabled=*/false, rtp)) {
+ // Flexfec not configured, but ulpfec is and is not disabled.
+ return std::make_unique<UlpfecGenerator>(
+ rtp.ulpfec.red_payload_type, rtp.ulpfec.ulpfec_payload_type, clock);
+ }
+
+ // Not a single FEC is given.
+ return nullptr;
+}
+
std::vector<RtpStreamSender> CreateRtpStreamSenders(
Clock* clock,
const RtpConfig& rtp_config,
@@ -121,7 +186,7 @@
Transport* send_transport,
RtcpBandwidthObserver* bandwidth_callback,
RtpTransportControllerSendInterface* transport,
- FlexfecSender* flexfec_sender,
+ const std::map<uint32_t, RtpState>& suspended_ssrcs,
RtcEventLog* event_log,
RateLimiter* retransmission_rate_limiter,
OverheadObserver* overhead_observer,
@@ -161,18 +226,17 @@
configuration.rtcp_report_interval_ms = rtcp_report_interval_ms;
std::vector<RtpStreamSender> rtp_streams;
- const std::vector<uint32_t>& flexfec_protected_ssrcs =
- rtp_config.flexfec.protected_media_ssrcs;
+
RTC_DCHECK(rtp_config.rtx.ssrcs.empty() ||
rtp_config.rtx.ssrcs.size() == rtp_config.rtx.ssrcs.size());
for (size_t i = 0; i < rtp_config.ssrcs.size(); ++i) {
+ RTPSenderVideo::Config video_config;
configuration.local_media_ssrc = rtp_config.ssrcs[i];
- bool enable_flexfec = flexfec_sender != nullptr &&
- std::find(flexfec_protected_ssrcs.begin(),
- flexfec_protected_ssrcs.end(),
- configuration.local_media_ssrc) !=
- flexfec_protected_ssrcs.end();
- configuration.flexfec_sender = enable_flexfec ? flexfec_sender : nullptr;
+
+ std::unique_ptr<VideoFecGenerator> fec_generator =
+ MaybeCreateFecGenerator(clock, rtp_config, suspended_ssrcs, i);
+ configuration.fec_generator = fec_generator.get();
+ video_config.fec_generator = fec_generator.get();
if (rtp_config.rtx.ssrcs.size() > i) {
configuration.rtx_send_ssrc = rtp_config.rtx.ssrcs[i];
@@ -188,76 +252,31 @@
rtp_rtcp->SetStorePacketsStatus(true, kMinSendSidePacketHistorySize);
FieldTrialBasedConfig field_trial_config;
- RTPSenderVideo::Config video_config;
video_config.clock = configuration.clock;
video_config.rtp_sender = rtp_rtcp->RtpSender();
- video_config.flexfec_sender = configuration.flexfec_sender;
video_config.frame_encryptor = frame_encryptor;
video_config.require_frame_encryption =
crypto_options.sframe.require_frame_encryption;
video_config.enable_retransmit_all_layers = false;
video_config.field_trials = &field_trial_config;
+
+ const bool using_flexfec =
+ fec_generator &&
+ fec_generator->GetFecType() == VideoFecGenerator::FecType::kFlexFec;
const bool should_disable_red_and_ulpfec =
- ShouldDisableRedAndUlpfec(enable_flexfec, rtp_config);
- if (rtp_config.ulpfec.red_payload_type != -1 &&
- !should_disable_red_and_ulpfec) {
+ ShouldDisableRedAndUlpfec(using_flexfec, rtp_config);
+ if (!should_disable_red_and_ulpfec &&
+ rtp_config.ulpfec.red_payload_type != -1) {
video_config.red_payload_type = rtp_config.ulpfec.red_payload_type;
}
- if (rtp_config.ulpfec.ulpfec_payload_type != -1 &&
- !should_disable_red_and_ulpfec) {
- video_config.ulpfec_payload_type = rtp_config.ulpfec.ulpfec_payload_type;
- }
video_config.frame_transformer = std::move(frame_transformer);
auto sender_video = std::make_unique<RTPSenderVideo>(video_config);
- rtp_streams.emplace_back(std::move(rtp_rtcp), std::move(sender_video));
+ rtp_streams.emplace_back(std::move(rtp_rtcp), std::move(sender_video),
+ std::move(fec_generator));
}
return rtp_streams;
}
-// TODO(brandtr): Update this function when we support multistream protection.
-std::unique_ptr<FlexfecSender> MaybeCreateFlexfecSender(
- Clock* clock,
- const RtpConfig& rtp,
- const std::map<uint32_t, RtpState>& suspended_ssrcs) {
- if (rtp.flexfec.payload_type < 0) {
- return nullptr;
- }
- RTC_DCHECK_GE(rtp.flexfec.payload_type, 0);
- RTC_DCHECK_LE(rtp.flexfec.payload_type, 127);
- if (rtp.flexfec.ssrc == 0) {
- RTC_LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. "
- "Therefore disabling FlexFEC.";
- return nullptr;
- }
- if (rtp.flexfec.protected_media_ssrcs.empty()) {
- RTC_LOG(LS_WARNING)
- << "FlexFEC is enabled, but no protected media SSRC given. "
- "Therefore disabling FlexFEC.";
- return nullptr;
- }
-
- if (rtp.flexfec.protected_media_ssrcs.size() > 1) {
- RTC_LOG(LS_WARNING)
- << "The supplied FlexfecConfig contained multiple protected "
- "media streams, but our implementation currently only "
- "supports protecting a single media stream. "
- "To avoid confusion, disabling FlexFEC completely.";
- return nullptr;
- }
-
- const RtpState* rtp_state = nullptr;
- auto it = suspended_ssrcs.find(rtp.flexfec.ssrc);
- if (it != suspended_ssrcs.end()) {
- rtp_state = &it->second;
- }
-
- RTC_DCHECK_EQ(1U, rtp.flexfec.protected_media_ssrcs.size());
- return std::make_unique<FlexfecSender>(
- rtp.flexfec.payload_type, rtp.flexfec.ssrc,
- rtp.flexfec.protected_media_ssrcs[0], rtp.mid, rtp.extensions,
- RTPSender::FecExtensionSizes(), rtp_state, clock);
-}
-
DataRate CalculateOverheadRate(DataRate data_rate,
DataSize packet_size,
DataSize overhead_per_packet) {
@@ -305,8 +324,6 @@
active_(false),
module_process_thread_(nullptr),
suspended_ssrcs_(std::move(suspended_ssrcs)),
- flexfec_sender_(
- MaybeCreateFlexfecSender(clock, rtp_config, suspended_ssrcs_)),
fec_controller_(std::move(fec_controller)),
fec_allowed_(true),
rtp_streams_(CreateRtpStreamSenders(clock,
@@ -316,7 +333,7 @@
send_transport,
transport->GetBandwidthObserver(),
transport,
- flexfec_sender_.get(),
+ suspended_ssrcs_,
event_log,
retransmission_limiter,
this,
@@ -379,6 +396,7 @@
}
}
+ bool fec_enabled = false;
for (const RtpStreamSender& stream : rtp_streams_) {
// Simulcast has one module for each layer. Set the CNAME on all modules.
stream.rtp_rtcp->SetCNAME(rtp_config_.c_name.c_str());
@@ -388,10 +406,13 @@
stream.rtp_rtcp->SetMaxRtpPacketSize(rtp_config_.max_packet_size);
stream.rtp_rtcp->RegisterSendPayloadFrequency(rtp_config_.payload_type,
kVideoPayloadTypeFrequency);
+ if (stream.fec_generator != nullptr) {
+ fec_enabled = true;
+ }
}
// Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic,
// so enable that logic if either of those FEC schemes are enabled.
- fec_controller_->SetProtectionMethod(FecEnabled(), NackEnabled());
+ fec_controller_->SetProtectionMethod(fec_enabled, NackEnabled());
fec_controller_->SetProtectionCallback(this);
// Signal congestion controller this object is ready for OnPacket* callbacks.
@@ -559,14 +580,6 @@
}
}
-bool RtpVideoSender::FecEnabled() const {
- const bool flexfec_enabled = (flexfec_sender_ != nullptr);
- const bool ulpfec_enabled =
- !webrtc::field_trial::IsEnabled("WebRTC-DisableUlpFecExperiment") &&
- (rtp_config_.ulpfec.ulpfec_payload_type >= 0);
- return flexfec_enabled || ulpfec_enabled;
-}
-
bool RtpVideoSender::NackEnabled() const {
const bool nack_enabled = rtp_config_.nack.rtp_history_ms > 0;
return nack_enabled;
@@ -661,6 +674,14 @@
uint32_t ssrc = rtp_config_.ssrcs[i];
RTC_DCHECK_EQ(ssrc, rtp_streams_[i].rtp_rtcp->SSRC());
rtp_states[ssrc] = rtp_streams_[i].rtp_rtcp->GetRtpState();
+
+ VideoFecGenerator* fec_generator = rtp_streams_[i].fec_generator.get();
+ if (fec_generator &&
+ fec_generator->GetFecType() == VideoFecGenerator::FecType::kFlexFec) {
+ auto* flexfec_sender = static_cast<FlexfecSender*>(fec_generator);
+ uint32_t ssrc = rtp_config_.flexfec.ssrc;
+ rtp_states[ssrc] = flexfec_sender->GetRtpState();
+ }
}
for (size_t i = 0; i < rtp_config_.rtx.ssrcs.size(); ++i) {
@@ -668,11 +689,6 @@
rtp_states[ssrc] = rtp_streams_[i].rtp_rtcp->GetRtxState();
}
- if (flexfec_sender_) {
- uint32_t ssrc = rtp_config_.flexfec.ssrc;
- rtp_states[ssrc] = flexfec_sender_->GetRtpState();
- }
-
return rtp_states;
}
diff --git a/call/rtp_video_sender.h b/call/rtp_video_sender.h
index 6c941f8..f7ebefc 100644
--- a/call/rtp_video_sender.h
+++ b/call/rtp_video_sender.h
@@ -51,7 +51,8 @@
// RtpVideoSender.
struct RtpStreamSender {
RtpStreamSender(std::unique_ptr<RtpRtcp> rtp_rtcp,
- std::unique_ptr<RTPSenderVideo> sender_video);
+ std::unique_ptr<RTPSenderVideo> sender_video,
+ std::unique_ptr<VideoFecGenerator> fec_generator);
~RtpStreamSender();
RtpStreamSender(RtpStreamSender&&) = default;
@@ -60,6 +61,7 @@
// Note: Needs pointer stability.
std::unique_ptr<RtpRtcp> rtp_rtcp;
std::unique_ptr<RTPSenderVideo> sender_video;
+ std::unique_ptr<VideoFecGenerator> fec_generator;
};
} // namespace webrtc_internal_rtp_video_sender
@@ -155,7 +157,6 @@
void ConfigureProtection();
void ConfigureSsrcs();
void ConfigureRids();
- bool FecEnabled() const;
bool NackEnabled() const;
uint32_t GetPacketizationOverheadRate() const;
@@ -173,8 +174,6 @@
rtc::ThreadChecker module_process_thread_checker_;
std::map<uint32_t, RtpState> suspended_ssrcs_;
- std::unique_ptr<FlexfecSender> flexfec_sender_;
-
const std::unique_ptr<FecController> fec_controller_;
bool fec_allowed_ RTC_GUARDED_BY(crit_);
diff --git a/modules/include/module_fec_types.h b/modules/include/module_fec_types.h
index 25d6bc5..f9b35cc 100644
--- a/modules/include/module_fec_types.h
+++ b/modules/include/module_fec_types.h
@@ -24,9 +24,9 @@
// Struct containing forward error correction settings.
struct FecProtectionParams {
- int fec_rate;
- int max_fec_frames;
- FecMaskType fec_mask_type;
+ int fec_rate = 0;
+ int max_fec_frames = 0;
+ FecMaskType fec_mask_type = FecMaskType::kFecMaskRandom;
};
} // namespace webrtc
diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn
index 19a2c13..2826d0f 100644
--- a/modules/rtp_rtcp/BUILD.gn
+++ b/modules/rtp_rtcp/BUILD.gn
@@ -214,6 +214,7 @@
"source/ulpfec_header_reader_writer.h",
"source/ulpfec_receiver_impl.cc",
"source/ulpfec_receiver_impl.h",
+ "source/video_fec_generator.h",
"source/video_rtp_depacketizer.h",
"source/video_rtp_depacketizer_av1.cc",
"source/video_rtp_depacketizer_av1.h",
diff --git a/modules/rtp_rtcp/include/flexfec_sender.h b/modules/rtp_rtcp/include/flexfec_sender.h
index 94f3502..4cc8f99 100644
--- a/modules/rtp_rtcp/include/flexfec_sender.h
+++ b/modules/rtp_rtcp/include/flexfec_sender.h
@@ -21,7 +21,9 @@
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/rtp_rtcp/source/rtp_header_extension_size.h"
#include "modules/rtp_rtcp/source/ulpfec_generator.h"
+#include "modules/rtp_rtcp/source/video_fec_generator.h"
#include "rtc_base/random.h"
+#include "rtc_base/rate_statistics.h"
namespace webrtc {
@@ -31,7 +33,7 @@
// Note that this class is not thread safe, and thus requires external
// synchronization. Currently, this is done using the lock in PayloadRouter.
-class FlexfecSender {
+class FlexfecSender : public VideoFecGenerator {
public:
FlexfecSender(int payload_type,
uint32_t ssrc,
@@ -43,26 +45,28 @@
Clock* clock);
~FlexfecSender();
- uint32_t ssrc() const { return ssrc_; }
+ FecType GetFecType() const override {
+ return VideoFecGenerator::FecType::kFlexFec;
+ }
+ absl::optional<uint32_t> FecSsrc() override { return ssrc_; }
// Sets the FEC rate, max frames sent before FEC packets are sent,
// and what type of generator matrices are used.
- void SetFecParameters(const FecProtectionParams& params);
+ void SetProtectionParameters(const FecProtectionParams& delta_params,
+ const FecProtectionParams& key_params) override;
// Adds a media packet to the internal buffer. When enough media packets
// have been added, the FEC packets are generated and stored internally.
// These FEC packets are then obtained by calling GetFecPackets().
- // Returns true if the media packet was successfully added.
- bool AddRtpPacketAndGenerateFec(const RtpPacketToSend& packet);
-
- // Returns true if there are generated FEC packets available.
- bool FecAvailable() const;
+ void AddPacketAndGenerateFec(const RtpPacketToSend& packet) override;
// Returns generated FlexFEC packets.
- std::vector<std::unique_ptr<RtpPacketToSend>> GetFecPackets();
+ std::vector<std::unique_ptr<RtpPacketToSend>> GetFecPackets() override;
// Returns the overhead, per packet, for FlexFEC.
- size_t MaxPacketOverhead() const;
+ size_t MaxPacketOverhead() const override;
+
+ DataRate CurrentFecRate() const override;
// Only called on the VideoSendStream queue, after operation has shut down.
RtpState GetRtpState();
@@ -87,6 +91,9 @@
UlpfecGenerator ulpfec_generator_;
const RtpHeaderExtensionMap rtp_header_extension_map_;
const size_t header_extensions_size_;
+
+ rtc::CriticalSection crit_;
+ RateStatistics fec_bitrate_ RTC_GUARDED_BY(crit_);
};
} // namespace webrtc
diff --git a/modules/rtp_rtcp/include/rtp_rtcp.h b/modules/rtp_rtcp/include/rtp_rtcp.h
index bb99b1a..e897718 100644
--- a/modules/rtp_rtcp/include/rtp_rtcp.h
+++ b/modules/rtp_rtcp/include/rtp_rtcp.h
@@ -24,13 +24,13 @@
#include "api/transport/webrtc_key_value_config.h"
#include "api/video/video_bitrate_allocation.h"
#include "modules/include/module.h"
-#include "modules/rtp_rtcp/include/flexfec_sender.h"
#include "modules/rtp_rtcp/include/receive_statistics.h"
#include "modules/rtp_rtcp/include/report_block_data.h"
#include "modules/rtp_rtcp/include/rtp_packet_sender.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h"
+#include "modules/rtp_rtcp/source/video_fec_generator.h"
#include "rtc_base/constructor_magic.h"
#include "rtc_base/deprecation.h"
@@ -94,9 +94,9 @@
// Spread any bursts of packets into smaller bursts to minimize packet loss.
RtpPacketSender* paced_sender = nullptr;
- // Generate FlexFEC packets.
- // TODO(brandtr): Remove when FlexfecSender is wired up to PacedSender.
- FlexfecSender* flexfec_sender = nullptr;
+ // Generates FEC packets.
+ // TODO(sprang): Wire up to RtpSenderEgress.
+ VideoFecGenerator* fec_generator = nullptr;
BitrateStatisticsObserver* send_bitrate_observer = nullptr;
SendSideDelayObserver* send_side_delay_observer = nullptr;
diff --git a/modules/rtp_rtcp/source/flexfec_sender.cc b/modules/rtp_rtcp/source/flexfec_sender.cc
index de0d412..4ff0893 100644
--- a/modules/rtp_rtcp/source/flexfec_sender.cc
+++ b/modules/rtp_rtcp/source/flexfec_sender.cc
@@ -91,11 +91,13 @@
seq_num_(rtp_state ? rtp_state->sequence_number
: random_.Rand(1, kMaxInitRtpSeqNumber)),
ulpfec_generator_(
- ForwardErrorCorrection::CreateFlexfec(ssrc, protected_media_ssrc)),
+ ForwardErrorCorrection::CreateFlexfec(ssrc, protected_media_ssrc),
+ clock_),
rtp_header_extension_map_(
RegisterSupportedExtensions(rtp_header_extensions)),
header_extensions_size_(
- RtpHeaderExtensionSize(extension_sizes, rtp_header_extension_map_)) {
+ RtpHeaderExtensionSize(extension_sizes, rtp_header_extension_map_)),
+ fec_bitrate_(/*max_window_size_ms=*/1000, RateStatistics::kBpsScale) {
// This object should not have been instantiated if FlexFEC is disabled.
RTC_DCHECK_GE(payload_type, 0);
RTC_DCHECK_LE(payload_type, 127);
@@ -105,30 +107,30 @@
// We are reusing the implementation from UlpfecGenerator for SetFecParameters,
// AddRtpPacketAndGenerateFec, and FecAvailable.
-void FlexfecSender::SetFecParameters(const FecProtectionParams& params) {
- ulpfec_generator_.SetFecParameters(params);
+void FlexfecSender::SetProtectionParameters(
+ const FecProtectionParams& delta_params,
+ const FecProtectionParams& key_params) {
+ ulpfec_generator_.SetProtectionParameters(delta_params, key_params);
}
-bool FlexfecSender::AddRtpPacketAndGenerateFec(const RtpPacketToSend& packet) {
+void FlexfecSender::AddPacketAndGenerateFec(const RtpPacketToSend& packet) {
// TODO(brandtr): Generalize this SSRC check when we support multistream
// protection.
RTC_DCHECK_EQ(packet.Ssrc(), protected_media_ssrc_);
- return ulpfec_generator_.AddRtpPacketAndGenerateFec(
- packet.Buffer(), packet.headers_size()) == 0;
-}
-
-bool FlexfecSender::FecAvailable() const {
- return ulpfec_generator_.FecAvailable();
+ ulpfec_generator_.AddPacketAndGenerateFec(packet);
}
std::vector<std::unique_ptr<RtpPacketToSend>> FlexfecSender::GetFecPackets() {
+ RTC_CHECK_RUNS_SERIALIZED(&ulpfec_generator_.race_checker_);
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets_to_send;
fec_packets_to_send.reserve(ulpfec_generator_.generated_fec_packets_.size());
+ size_t total_fec_data_bytes = 0;
for (const auto* fec_packet : ulpfec_generator_.generated_fec_packets_) {
std::unique_ptr<RtpPacketToSend> fec_packet_to_send(
new RtpPacketToSend(&rtp_header_extension_map_));
fec_packet_to_send->set_packet_type(
RtpPacketMediaType::kForwardErrorCorrection);
+ fec_packet_to_send->set_allow_retransmission(false);
// RTP header.
fec_packet_to_send->SetMarker(false);
@@ -157,9 +159,13 @@
fec_packet_to_send->AllocatePayload(fec_packet->data.size());
memcpy(payload, fec_packet->data.cdata(), fec_packet->data.size());
+ total_fec_data_bytes += fec_packet_to_send->size();
fec_packets_to_send.push_back(std::move(fec_packet_to_send));
}
- ulpfec_generator_.ResetState();
+
+ if (!fec_packets_to_send.empty()) {
+ ulpfec_generator_.ResetState();
+ }
int64_t now_ms = clock_->TimeInMilliseconds();
if (!fec_packets_to_send.empty() &&
@@ -170,6 +176,9 @@
last_generated_packet_ms_ = now_ms;
}
+ rtc::CritScope cs(&crit_);
+ fec_bitrate_.Update(total_fec_data_bytes, now_ms);
+
return fec_packets_to_send;
}
@@ -178,6 +187,12 @@
return header_extensions_size_ + kFlexfecMaxHeaderSize;
}
+DataRate FlexfecSender::CurrentFecRate() const {
+ rtc::CritScope cs(&crit_);
+ return DataRate::BitsPerSec(
+ fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0));
+}
+
RtpState FlexfecSender::GetRtpState() {
RtpState rtp_state;
rtp_state.sequence_number = seq_num_;
diff --git a/modules/rtp_rtcp/source/flexfec_sender_unittest.cc b/modules/rtp_rtcp/source/flexfec_sender_unittest.cc
index 10ec2e7..e4501c2 100644
--- a/modules/rtp_rtcp/source/flexfec_sender_unittest.cc
+++ b/modules/rtp_rtcp/source/flexfec_sender_unittest.cc
@@ -55,7 +55,7 @@
params.fec_mask_type = kFecMaskRandom;
constexpr size_t kNumPackets = 4;
- sender->SetFecParameters(params);
+ sender->SetProtectionParameters(params, params);
AugmentedPacketGenerator packet_generator(kMediaSsrc);
packet_generator.NewFrame(kNumPackets);
for (size_t i = 0; i < kNumPackets; ++i) {
@@ -63,13 +63,12 @@
packet_generator.NextPacket(i, kPayloadLength);
RtpPacketToSend rtp_packet(nullptr); // No header extensions.
rtp_packet.Parse(packet->data);
- EXPECT_TRUE(sender->AddRtpPacketAndGenerateFec(rtp_packet));
+ sender->AddPacketAndGenerateFec(rtp_packet);
}
- EXPECT_TRUE(sender->FecAvailable());
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
sender->GetFecPackets();
- EXPECT_FALSE(sender->FecAvailable());
EXPECT_EQ(1U, fec_packets.size());
+ EXPECT_TRUE(sender->GetFecPackets().empty());
return std::move(fec_packets.front());
}
@@ -82,7 +81,7 @@
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
nullptr /* rtp_state */, &clock);
- EXPECT_EQ(kFlexfecSsrc, sender.ssrc());
+ EXPECT_EQ(kFlexfecSsrc, sender.FecSsrc());
}
TEST(FlexfecSenderTest, NoFecAvailableBeforeMediaAdded) {
@@ -91,9 +90,7 @@
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
nullptr /* rtp_state */, &clock);
- EXPECT_FALSE(sender.FecAvailable());
- auto fec_packets = sender.GetFecPackets();
- EXPECT_EQ(0U, fec_packets.size());
+ EXPECT_TRUE(sender.GetFecPackets().empty());
}
TEST(FlexfecSenderTest, ProtectOneFrameWithOneFecPacket) {
@@ -124,7 +121,7 @@
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc, kNoMid,
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
nullptr /* rtp_state */, &clock);
- sender.SetFecParameters(params);
+ sender.SetProtectionParameters(params, params);
AugmentedPacketGenerator packet_generator(kMediaSsrc);
for (size_t i = 0; i < kNumFrames; ++i) {
@@ -134,14 +131,13 @@
packet_generator.NextPacket(i, kPayloadLength);
RtpPacketToSend rtp_packet(nullptr);
rtp_packet.Parse(packet->data);
- EXPECT_TRUE(sender.AddRtpPacketAndGenerateFec(rtp_packet));
+ sender.AddPacketAndGenerateFec(rtp_packet);
}
}
- EXPECT_TRUE(sender.FecAvailable());
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
sender.GetFecPackets();
- EXPECT_FALSE(sender.FecAvailable());
ASSERT_EQ(1U, fec_packets.size());
+ EXPECT_TRUE(sender.GetFecPackets().empty());
RtpPacketToSend* fec_packet = fec_packets.front().get();
EXPECT_EQ(kRtpHeaderSize, fec_packet->headers_size());
@@ -164,7 +160,7 @@
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc, kNoMid,
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
nullptr /* rtp_state */, &clock);
- sender.SetFecParameters(params);
+ sender.SetProtectionParameters(params, params);
AugmentedPacketGenerator packet_generator(kMediaSsrc);
for (size_t i = 0; i < kNumFrames; ++i) {
@@ -174,13 +170,12 @@
packet_generator.NextPacket(i, kPayloadLength);
RtpPacketToSend rtp_packet(nullptr);
rtp_packet.Parse(packet->data);
- EXPECT_TRUE(sender.AddRtpPacketAndGenerateFec(rtp_packet));
+ sender.AddPacketAndGenerateFec(rtp_packet);
}
- EXPECT_TRUE(sender.FecAvailable());
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
sender.GetFecPackets();
- EXPECT_FALSE(sender.FecAvailable());
ASSERT_EQ(1U, fec_packets.size());
+ EXPECT_TRUE(sender.GetFecPackets().empty());
RtpPacketToSend* fec_packet = fec_packets.front().get();
EXPECT_EQ(kRtpHeaderSize, fec_packet->headers_size());
diff --git a/modules/rtp_rtcp/source/rtcp_receiver.cc b/modules/rtp_rtcp/source/rtcp_receiver.cc
index 6b64473..26465ad 100644
--- a/modules/rtp_rtcp/source/rtcp_receiver.cc
+++ b/modules/rtp_rtcp/source/rtcp_receiver.cc
@@ -72,8 +72,11 @@
if (config.rtx_send_ssrc) {
ssrcs.insert(*config.rtx_send_ssrc);
}
- if (config.flexfec_sender) {
- ssrcs.insert(config.flexfec_sender->ssrc());
+ if (config.fec_generator) {
+ absl::optional<uint32_t> flexfec_ssrc = config.fec_generator->FecSsrc();
+ if (flexfec_ssrc) {
+ ssrcs.insert(*flexfec_ssrc);
+ }
}
return ssrcs;
}
diff --git a/modules/rtp_rtcp/source/rtp_packet_to_send.h b/modules/rtp_rtcp/source/rtp_packet_to_send.h
index 57493e3..8997bce 100644
--- a/modules/rtp_rtcp/source/rtp_packet_to_send.h
+++ b/modules/rtp_rtcp/source/rtp_packet_to_send.h
@@ -98,12 +98,16 @@
VideoTimingExtension::kNetwork2TimestampDeltaOffset);
}
+ // Indicates if packet is the first packet of a video frame.
void set_first_packet_of_frame(bool is_first_packet) {
is_first_packet_of_frame_ = is_first_packet;
}
-
bool is_first_packet_of_frame() const { return is_first_packet_of_frame_; }
+ // Indicates if packet contains payload for a video key-frame.
+ void set_is_key_frame(bool is_key_frame) { is_key_frame_ = is_key_frame; }
+ bool is_key_frame() const { return is_key_frame_; }
+
private:
int64_t capture_time_ms_ = 0;
absl::optional<RtpPacketMediaType> packet_type_;
@@ -111,6 +115,7 @@
absl::optional<uint16_t> retransmitted_sequence_number_;
std::vector<uint8_t> application_data_;
bool is_first_packet_of_frame_ = false;
+ bool is_key_frame_ = false;
};
} // namespace webrtc
diff --git a/modules/rtp_rtcp/source/rtp_sender.cc b/modules/rtp_rtcp/source/rtp_sender.cc
index 3277c67..c48a662 100644
--- a/modules/rtp_rtcp/source/rtp_sender.cc
+++ b/modules/rtp_rtcp/source/rtp_sender.cc
@@ -102,9 +102,8 @@
audio_configured_(config.audio),
ssrc_(config.local_media_ssrc),
rtx_ssrc_(config.rtx_send_ssrc),
- flexfec_ssrc_(config.flexfec_sender
- ? absl::make_optional(config.flexfec_sender->ssrc())
- : absl::nullopt),
+ flexfec_ssrc_(config.fec_generator ? config.fec_generator->FecSsrc()
+ : absl::nullopt),
packet_history_(packet_history),
paced_sender_(packet_sender),
sending_media_(true), // Default to sending media.
diff --git a/modules/rtp_rtcp/source/rtp_sender_egress.cc b/modules/rtp_rtcp/source/rtp_sender_egress.cc
index d34d7c6..ec546c4 100644
--- a/modules/rtp_rtcp/source/rtp_sender_egress.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_egress.cc
@@ -57,9 +57,8 @@
RtpPacketHistory* packet_history)
: ssrc_(config.local_media_ssrc),
rtx_ssrc_(config.rtx_send_ssrc),
- flexfec_ssrc_(config.flexfec_sender
- ? absl::make_optional(config.flexfec_sender->ssrc())
- : absl::nullopt),
+ flexfec_ssrc_(config.fec_generator ? config.fec_generator->FecSsrc()
+ : absl::nullopt),
populate_network2_timestamp_(config.populate_network2_timestamp),
send_side_bwe_with_overhead_(
IsEnabled("WebRTC-SendSideBwe-WithOverhead", config.field_trials)),
diff --git a/modules/rtp_rtcp/source/rtp_sender_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_unittest.cc
index c3ae539..3b85166 100644
--- a/modules/rtp_rtcp/source/rtp_sender_unittest.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_unittest.cc
@@ -272,7 +272,7 @@
config.outgoing_transport = &transport_;
config.local_media_ssrc = kSsrc;
config.rtx_send_ssrc = kRtxSsrc;
- config.flexfec_sender = &flexfec_sender_;
+ config.fec_generator = &flexfec_sender_;
config.event_log = &mock_rtc_event_log_;
config.send_packet_observer = &send_packet_observer_;
config.retransmission_rate_limiter = &retransmission_rate_limiter_;
@@ -1225,7 +1225,7 @@
config.outgoing_transport = &transport_;
config.paced_sender = &mock_paced_sender_;
config.local_media_ssrc = kSsrc;
- config.flexfec_sender = &flexfec_sender_;
+ config.fec_generator = &flexfec_sender_;
config.event_log = &mock_rtc_event_log_;
config.send_packet_observer = &send_packet_observer_;
config.retransmission_rate_limiter = &retransmission_rate_limiter_;
@@ -1239,7 +1239,7 @@
RTPSenderVideo::Config video_config;
video_config.clock = &fake_clock_;
video_config.rtp_sender = rtp_sender();
- video_config.flexfec_sender = &flexfec_sender;
+ video_config.fec_generator = &flexfec_sender;
video_config.field_trials = &field_trials;
RTPSenderVideo rtp_sender_video(video_config);
@@ -1311,7 +1311,7 @@
config.clock = &fake_clock_;
config.outgoing_transport = &transport_;
config.local_media_ssrc = kSsrc;
- config.flexfec_sender = &flexfec_sender;
+ config.fec_generator = &flexfec_sender;
config.event_log = &mock_rtc_event_log_;
config.send_packet_observer = &send_packet_observer_;
config.retransmission_rate_limiter = &retransmission_rate_limiter_;
@@ -1323,7 +1323,7 @@
RTPSenderVideo::Config video_config;
video_config.clock = &fake_clock_;
video_config.rtp_sender = rtp_sender();
- video_config.flexfec_sender = &flexfec_sender;
+ video_config.fec_generator = &flexfec_sender;
video_config.field_trials = &field_trials;
RTPSenderVideo rtp_sender_video(video_config);
@@ -1583,7 +1583,7 @@
config.outgoing_transport = &transport_;
config.paced_sender = &mock_paced_sender_;
config.local_media_ssrc = kSsrc;
- config.flexfec_sender = &flexfec_sender;
+ config.fec_generator = &flexfec_sender;
config.event_log = &mock_rtc_event_log_;
config.send_packet_observer = &send_packet_observer_;
config.retransmission_rate_limiter = &retransmission_rate_limiter_;
@@ -1595,7 +1595,7 @@
RTPSenderVideo::Config video_config;
video_config.clock = &fake_clock_;
video_config.rtp_sender = rtp_sender();
- video_config.flexfec_sender = &flexfec_sender;
+ video_config.fec_generator = &flexfec_sender;
video_config.field_trials = &field_trials;
RTPSenderVideo rtp_sender_video(video_config);
// Parameters selected to generate a single FEC packet per media packet.
@@ -1777,12 +1777,14 @@
const uint8_t kPayloadType = 127;
const VideoCodecType kCodecType = VideoCodecType::kVideoCodecGeneric;
FieldTrialBasedConfig field_trials;
+ UlpfecGenerator ulpfec_generator(kRedPayloadType, kUlpfecPayloadType,
+ &fake_clock_);
RTPSenderVideo::Config video_config;
video_config.clock = &fake_clock_;
video_config.rtp_sender = rtp_sender();
video_config.field_trials = &field_trials;
video_config.red_payload_type = kRedPayloadType;
- video_config.ulpfec_payload_type = kUlpfecPayloadType;
+ video_config.fec_generator = &ulpfec_generator;
RTPSenderVideo rtp_sender_video(video_config);
uint8_t payload[] = {47, 11, 32, 93, 89};
rtp_sender_context_->packet_history_.SetStorePacketsStatus(
@@ -2118,7 +2120,7 @@
config.outgoing_transport = &transport_;
config.local_media_ssrc = kSsrc;
config.rtx_send_ssrc = kRtxSsrc;
- config.flexfec_sender = &flexfec_sender_;
+ config.fec_generator = &flexfec_sender_;
config.send_side_delay_observer = &send_side_delay_observer;
config.event_log = &mock_rtc_event_log_;
config.send_packet_observer = &send_packet_observer_;
diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc
index 5ab7831..ec5cf8f 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video.cc
@@ -260,11 +260,7 @@
current_playout_delay_{-1, -1},
playout_delay_pending_(false),
red_payload_type_(config.red_payload_type),
- ulpfec_payload_type_(config.ulpfec_payload_type),
- flexfec_sender_(config.flexfec_sender),
- delta_fec_params_{0, 1, kFecMaskRandom},
- key_fec_params_{0, 1, kFecMaskRandom},
- fec_bitrate_(1000, RateStatistics::kBpsScale),
+ fec_generator_(config.fec_generator),
video_bitrate_(1000, RateStatistics::kBpsScale),
packetization_overhead_bitrate_(1000, RateStatistics::kBpsScale),
frame_encryptor_(config.frame_encryptor),
@@ -293,83 +289,6 @@
frame_transformer_delegate_->Reset();
}
-void RTPSenderVideo::AppendAsRedMaybeWithUlpfec(
- std::unique_ptr<RtpPacketToSend> media_packet,
- bool protect_media_packet,
- std::vector<std::unique_ptr<RtpPacketToSend>>* packets) {
- std::unique_ptr<RtpPacketToSend> red_packet(
- new RtpPacketToSend(*media_packet));
- BuildRedPayload(*media_packet, red_packet.get());
- red_packet->SetPayloadType(*red_payload_type_);
-
- std::vector<std::unique_ptr<RedPacket>> fec_packets;
- if (ulpfec_enabled()) {
- if (protect_media_packet) {
- if (exclude_transport_sequence_number_from_fec_experiment_) {
- // See comments at the top of the file why experiment
- // "WebRTC-kExcludeTransportSequenceNumberFromFec" is needed in
- // conjunction with datagram transport.
- // TODO(sukhanov): We may also need to implement it for flexfec_sender
- // if we decide to keep this approach in the future.
- uint16_t transport_senquence_number;
- if (media_packet->GetExtension<webrtc::TransportSequenceNumber>(
- &transport_senquence_number)) {
- if (!media_packet->RemoveExtension(
- webrtc::TransportSequenceNumber::kId)) {
- RTC_NOTREACHED()
- << "Failed to remove transport sequence number, packet="
- << media_packet->ToString();
- }
- }
- }
-
- ulpfec_generator_.AddRtpPacketAndGenerateFec(
- media_packet->Buffer(), media_packet->headers_size());
- }
- uint16_t num_fec_packets = ulpfec_generator_.NumAvailableFecPackets();
- if (num_fec_packets > 0) {
- uint16_t first_fec_sequence_number =
- rtp_sender_->AllocateSequenceNumber(num_fec_packets);
- fec_packets = ulpfec_generator_.GetUlpfecPacketsAsRed(
- *red_payload_type_, *ulpfec_payload_type_, first_fec_sequence_number);
- RTC_DCHECK_EQ(num_fec_packets, fec_packets.size());
- }
- }
-
- // Send |red_packet| instead of |packet| for allocated sequence number.
- red_packet->set_packet_type(RtpPacketMediaType::kVideo);
- red_packet->set_allow_retransmission(media_packet->allow_retransmission());
- packets->emplace_back(std::move(red_packet));
-
- for (const auto& fec_packet : fec_packets) {
- // TODO(danilchap): Make ulpfec_generator_ generate RtpPacketToSend to avoid
- // reparsing them.
- std::unique_ptr<RtpPacketToSend> rtp_packet(
- new RtpPacketToSend(*media_packet));
- RTC_CHECK(rtp_packet->Parse(fec_packet->data(), fec_packet->length()));
- rtp_packet->set_capture_time_ms(media_packet->capture_time_ms());
- rtp_packet->set_packet_type(RtpPacketMediaType::kForwardErrorCorrection);
- rtp_packet->set_allow_retransmission(false);
- RTC_DCHECK_EQ(fec_packet->length(), rtp_packet->size());
- packets->emplace_back(std::move(rtp_packet));
- }
-}
-
-void RTPSenderVideo::GenerateAndAppendFlexfec(
- std::vector<std::unique_ptr<RtpPacketToSend>>* packets) {
- RTC_DCHECK(flexfec_sender_);
-
- if (flexfec_sender_->FecAvailable()) {
- std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
- flexfec_sender_->GetFecPackets();
- for (auto& fec_packet : fec_packets) {
- fec_packet->set_packet_type(RtpPacketMediaType::kForwardErrorCorrection);
- fec_packet->set_allow_retransmission(false);
- packets->emplace_back(std::move(fec_packet));
- }
- }
-}
-
void RTPSenderVideo::LogAndSendToNetwork(
std::vector<std::unique_ptr<RtpPacketToSend>> packets,
size_t unpacketized_payload_size) {
@@ -388,16 +307,9 @@
rtc::CritScope cs(&stats_crit_);
size_t packetized_payload_size = 0;
for (const auto& packet : packets) {
- switch (*packet->packet_type()) {
- case RtpPacketMediaType::kVideo:
- video_bitrate_.Update(packet->size(), now_ms);
- packetized_payload_size += packet->payload_size();
- break;
- case RtpPacketMediaType::kForwardErrorCorrection:
- fec_bitrate_.Update(packet->size(), clock_->TimeInMilliseconds());
- break;
- default:
- continue;
+ if (*packet->packet_type() == RtpPacketMediaType::kVideo) {
+ video_bitrate_.Update(packet->size(), now_ms);
+ packetized_payload_size += packet->payload_size();
}
}
// AV1 packetizer may produce less packetized bytes than unpacketized.
@@ -412,39 +324,31 @@
}
size_t RTPSenderVideo::FecPacketOverhead() const {
- if (flexfec_enabled())
- return flexfec_sender_->MaxPacketOverhead();
-
- size_t overhead = 0;
+ size_t overhead = fec_generator_ ? fec_generator_->MaxPacketOverhead() : 0u;
if (red_enabled()) {
// The RED overhead is due to a small header.
overhead += kRedForFecHeaderLength;
- }
- if (ulpfec_enabled()) {
- // For ULPFEC, the overhead is the FEC headers plus RED for FEC header
- // (see above) plus anything in RTP header beyond the 12 bytes base header
- // (CSRC list, extensions...)
- // This reason for the header extensions to be included here is that
- // from an FEC viewpoint, they are part of the payload to be protected.
- // (The base RTP header is already protected by the FEC header.)
- overhead += ulpfec_generator_.MaxPacketOverhead() +
- (rtp_sender_->RtpHeaderLength() - kRtpHeaderSize);
+
+ // TODO(bugs.webrtc.org/11340): Move this into UlpfecGenerator.
+ if (fec_generator_ &&
+ fec_generator_->GetFecType() == VideoFecGenerator::FecType::kUlpFec) {
+ // For ULPFEC, the overhead is the FEC headers plus RED for FEC header
+ // (see above) plus anything in RTP header beyond the 12 bytes base header
+ // (CSRC list, extensions...)
+ // This reason for the header extensions to be included here is that
+ // from an FEC viewpoint, they are part of the payload to be protected.
+ // (The base RTP header is already protected by the FEC header.)
+ overhead += rtp_sender_->RtpHeaderLength() - kRtpHeaderSize;
+ }
}
return overhead;
}
void RTPSenderVideo::SetFecParameters(const FecProtectionParams& delta_params,
const FecProtectionParams& key_params) {
- rtc::CritScope cs(&crit_);
- delta_fec_params_ = delta_params;
- key_fec_params_ = key_params;
-}
-
-absl::optional<uint32_t> RTPSenderVideo::FlexfecSsrc() const {
- if (flexfec_sender_) {
- return flexfec_sender_->ssrc();
+ if (fec_generator_) {
+ fec_generator_->SetProtectionParameters(delta_params, key_params);
}
- return absl::nullopt;
}
void RTPSenderVideo::SetVideoStructure(
@@ -565,19 +469,6 @@
transmit_color_space_next_frame_ ? !IsBaseLayer(video_header) : false;
}
- if (flexfec_enabled() || ulpfec_enabled()) {
- rtc::CritScope cs(&crit_);
- // FEC settings.
- const FecProtectionParams& fec_params =
- video_header.frame_type == VideoFrameType::kVideoFrameKey
- ? key_fec_params_
- : delta_fec_params_;
- if (flexfec_enabled())
- flexfec_sender_->SetFecParameters(fec_params);
- if (ulpfec_enabled())
- ulpfec_generator_.SetFecParameters(fec_params);
- }
-
// Maximum size of packet including rtp headers.
// Extra space left in case packet will be resent using fec or rtx.
int packet_capacity = rtp_sender_->MaxRtpPacketSize() - FecPacketOverhead() -
@@ -769,21 +660,40 @@
packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds());
}
+ if (protect_packet && fec_generator_) {
+ if (red_enabled() &&
+ exclude_transport_sequence_number_from_fec_experiment_) {
+ // See comments at the top of the file why experiment
+ // "WebRTC-kExcludeTransportSequenceNumberFromFec" is needed in
+ // conjunction with datagram transport.
+ // TODO(sukhanov): We may also need to implement it for flexfec_sender
+ // if we decide to keep this approach in the future.
+ uint16_t transport_senquence_number;
+ if (packet->GetExtension<webrtc::TransportSequenceNumber>(
+ &transport_senquence_number)) {
+ if (!packet->RemoveExtension(webrtc::TransportSequenceNumber::kId)) {
+ RTC_NOTREACHED()
+ << "Failed to remove transport sequence number, packet="
+ << packet->ToString();
+ }
+ }
+ }
+
+ fec_generator_->AddPacketAndGenerateFec(*packet);
+ }
+
if (red_enabled()) {
- AppendAsRedMaybeWithUlpfec(std::move(packet), protect_packet,
- &rtp_packets);
+ std::unique_ptr<RtpPacketToSend> red_packet(new RtpPacketToSend(*packet));
+ BuildRedPayload(*packet, red_packet.get());
+ red_packet->SetPayloadType(*red_payload_type_);
+
+ // Send |red_packet| instead of |packet| for allocated sequence number.
+ red_packet->set_packet_type(RtpPacketMediaType::kVideo);
+ red_packet->set_allow_retransmission(packet->allow_retransmission());
+ rtp_packets.emplace_back(std::move(red_packet));
} else {
packet->set_packet_type(RtpPacketMediaType::kVideo);
- const RtpPacketToSend& media_packet = *packet;
rtp_packets.emplace_back(std::move(packet));
- if (flexfec_enabled()) {
- // TODO(brandtr): Remove the FlexFEC code path when FlexfecSender
- // is wired up to PacedSender instead.
- if (protect_packet) {
- flexfec_sender_->AddRtpPacketAndGenerateFec(media_packet);
- }
- GenerateAndAppendFlexfec(&rtp_packets);
- }
}
if (first_frame) {
@@ -798,6 +708,22 @@
}
}
+ if (fec_generator_) {
+ // Fetch any FEC packets generated from the media frame and add them to
+ // the list of packets to send.
+ auto fec_packets = fec_generator_->GetFecPackets();
+
+ // TODO(bugs.webrtc.org/11340): Move sequence number assignment into
+ // UlpfecGenerator.
+ const bool generate_sequence_numbers = !fec_generator_->FecSsrc();
+ for (auto& fec_packet : fec_packets) {
+ if (generate_sequence_numbers) {
+ rtp_sender_->AssignSequenceNumber(fec_packet.get());
+ }
+ rtp_packets.emplace_back(std::move(fec_packet));
+ }
+ }
+
LogAndSendToNetwork(std::move(rtp_packets), unpacketized_payload_size);
TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp",
@@ -830,8 +756,7 @@
}
uint32_t RTPSenderVideo::FecOverheadRate() const {
- rtc::CritScope cs(&stats_crit_);
- return fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0);
+ return fec_generator_ ? fec_generator_->CurrentFecRate().bps<uint32_t>() : 0u;
}
uint32_t RTPSenderVideo::PacketizationOverheadBps() const {
diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h
index 5fb6699..440a060 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video.h
+++ b/modules/rtp_rtcp/source/rtp_sender_video.h
@@ -24,14 +24,13 @@
#include "api/video/video_codec_type.h"
#include "api/video/video_frame_type.h"
#include "modules/include/module_common_types.h"
-#include "modules/rtp_rtcp/include/flexfec_sender.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/rtp_rtcp/source/absolute_capture_time_sender.h"
#include "modules/rtp_rtcp/source/rtp_rtcp_config.h"
#include "modules/rtp_rtcp/source/rtp_sender.h"
#include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h"
#include "modules/rtp_rtcp/source/rtp_video_header.h"
-#include "modules/rtp_rtcp/source/ulpfec_generator.h"
+#include "modules/rtp_rtcp/source/video_fec_generator.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/one_time_event.h"
#include "rtc_base/race_checker.h"
@@ -71,11 +70,11 @@
Clock* clock = nullptr;
RTPSender* rtp_sender = nullptr;
FlexfecSender* flexfec_sender = nullptr;
+ VideoFecGenerator* fec_generator = nullptr;
FrameEncryptorInterface* frame_encryptor = nullptr;
bool require_frame_encryption = false;
bool enable_retransmit_all_layers = false;
absl::optional<int> red_payload_type;
- absl::optional<int> ulpfec_payload_type;
const WebRtcKeyValueConfig* field_trials = nullptr;
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer;
};
@@ -115,13 +114,9 @@
// FlexFEC/ULPFEC.
// Set FEC rates, max frames before FEC is sent, and type of FEC masks.
- // Returns false on failure.
void SetFecParameters(const FecProtectionParams& delta_params,
const FecProtectionParams& key_params);
- // FlexFEC.
- absl::optional<uint32_t> FlexfecSsrc() const;
-
uint32_t VideoBitrateSent() const;
uint32_t FecOverheadRate() const;
@@ -150,27 +145,12 @@
size_t FecPacketOverhead() const RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_);
- void AppendAsRedMaybeWithUlpfec(
- std::unique_ptr<RtpPacketToSend> media_packet,
- bool protect_media_packet,
- std::vector<std::unique_ptr<RtpPacketToSend>>* packets)
- RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_);
-
- // TODO(brandtr): Remove the FlexFEC functions when FlexfecSender has been
- // moved to PacedSender.
- void GenerateAndAppendFlexfec(
- std::vector<std::unique_ptr<RtpPacketToSend>>* packets);
-
void LogAndSendToNetwork(
std::vector<std::unique_ptr<RtpPacketToSend>> packets,
size_t unpacketized_payload_size);
bool red_enabled() const { return red_payload_type_.has_value(); }
- bool ulpfec_enabled() const { return ulpfec_payload_type_.has_value(); }
-
- bool flexfec_enabled() const { return flexfec_sender_ != nullptr; }
-
bool UpdateConditionalRetransmit(uint8_t temporal_id,
int64_t expected_retransmission_time_ms)
RTC_EXCLUSIVE_LOCKS_REQUIRED(stats_crit_);
@@ -201,22 +181,10 @@
// Should never be held when calling out of this class.
rtc::CriticalSection crit_;
- // RED/ULPFEC.
const absl::optional<int> red_payload_type_;
- const absl::optional<int> ulpfec_payload_type_;
- UlpfecGenerator ulpfec_generator_ RTC_GUARDED_BY(send_checker_);
-
- // FlexFEC.
- FlexfecSender* const flexfec_sender_;
-
- // FEC parameters, applicable to either ULPFEC or FlexFEC.
- FecProtectionParams delta_fec_params_ RTC_GUARDED_BY(crit_);
- FecProtectionParams key_fec_params_ RTC_GUARDED_BY(crit_);
+ VideoFecGenerator* const fec_generator_;
rtc::CriticalSection stats_crit_;
- // Bitrate used for FEC payload, RED headers, RTP headers for FEC packets
- // and any padding overhead.
- RateStatistics fec_bitrate_ RTC_GUARDED_BY(stats_crit_);
// Bitrate used for video payload and RTP headers.
RateStatistics video_bitrate_ RTC_GUARDED_BY(stats_crit_);
RateStatistics packetization_overhead_bitrate_ RTC_GUARDED_BY(stats_crit_);
diff --git a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
index a25c7b7..5be9c9e 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
@@ -131,7 +131,7 @@
Config config;
config.clock = clock;
config.rtp_sender = rtp_sender;
- config.flexfec_sender = flexfec_sender;
+ config.fec_generator = flexfec_sender;
config.field_trials = &field_trials;
return config;
}()) {}
diff --git a/modules/rtp_rtcp/source/ulpfec_generator.cc b/modules/rtp_rtcp/source/ulpfec_generator.cc
index 92e65df..265fa4d 100644
--- a/modules/rtp_rtcp/source/ulpfec_generator.cc
+++ b/modules/rtp_rtcp/source/ulpfec_generator.cc
@@ -22,6 +22,7 @@
#include "modules/rtp_rtcp/source/forward_error_correction_internal.h"
#include "modules/rtp_rtcp/source/rtp_utility.h"
#include "rtc_base/checks.h"
+#include "rtc_base/critical_section.h"
namespace webrtc {
@@ -62,128 +63,119 @@
} // namespace
-RedPacket::RedPacket(size_t length)
- : data_(new uint8_t[length]), length_(length), header_length_(0) {}
+UlpfecGenerator::Params::Params() = default;
+UlpfecGenerator::Params::Params(FecProtectionParams delta_params,
+ FecProtectionParams keyframe_params)
+ : delta_params(delta_params), keyframe_params(keyframe_params) {}
-RedPacket::~RedPacket() = default;
-
-void RedPacket::CreateHeader(const uint8_t* rtp_header,
- size_t header_length,
- int red_payload_type,
- int payload_type) {
- RTC_DCHECK_LE(header_length + kRedForFecHeaderLength, length_);
- memcpy(data_.get(), rtp_header, header_length);
- // Replace payload type.
- data_[1] &= 0x80;
- data_[1] += red_payload_type;
- // Add RED header
- // f-bit always 0
- data_[header_length] = static_cast<uint8_t>(payload_type);
- header_length_ = header_length + kRedForFecHeaderLength;
-}
-
-void RedPacket::SetSeqNum(int seq_num) {
- RTC_DCHECK_GE(seq_num, 0);
- RTC_DCHECK_LT(seq_num, 1 << 16);
-
- ByteWriter<uint16_t>::WriteBigEndian(&data_[2], seq_num);
-}
-
-void RedPacket::AssignPayload(const uint8_t* payload, size_t length) {
- RTC_DCHECK_LE(header_length_ + length, length_);
- memcpy(data_.get() + header_length_, payload, length);
-}
-
-void RedPacket::ClearMarkerBit() {
- data_[1] &= 0x7F;
-}
-
-uint8_t* RedPacket::data() const {
- return data_.get();
-}
-
-size_t RedPacket::length() const {
- return length_;
-}
-
-UlpfecGenerator::UlpfecGenerator()
- : UlpfecGenerator(ForwardErrorCorrection::CreateUlpfec(kUnknownSsrc)) {}
-
-UlpfecGenerator::UlpfecGenerator(std::unique_ptr<ForwardErrorCorrection> fec)
- : fec_(std::move(fec)),
- last_media_packet_rtp_header_length_(0),
+UlpfecGenerator::UlpfecGenerator(int red_payload_type,
+ int ulpfec_payload_type,
+ Clock* clock)
+ : red_payload_type_(red_payload_type),
+ ulpfec_payload_type_(ulpfec_payload_type),
+ clock_(clock),
+ fec_(ForwardErrorCorrection::CreateUlpfec(kUnknownSsrc)),
num_protected_frames_(0),
- min_num_media_packets_(1) {
- memset(¶ms_, 0, sizeof(params_));
- memset(&new_params_, 0, sizeof(new_params_));
-}
+ min_num_media_packets_(1),
+ keyframe_in_process_(false),
+ fec_bitrate_(/*max_window_size_ms=*/1000, RateStatistics::kBpsScale) {}
+
+// Used by FlexFecSender, payload types are unused.
+UlpfecGenerator::UlpfecGenerator(std::unique_ptr<ForwardErrorCorrection> fec,
+ Clock* clock)
+ : red_payload_type_(0),
+ ulpfec_payload_type_(0),
+ clock_(clock),
+ fec_(std::move(fec)),
+ num_protected_frames_(0),
+ min_num_media_packets_(1),
+ keyframe_in_process_(false),
+ fec_bitrate_(/*max_window_size_ms=*/1000, RateStatistics::kBpsScale) {}
UlpfecGenerator::~UlpfecGenerator() = default;
-void UlpfecGenerator::SetFecParameters(const FecProtectionParams& params) {
- RTC_DCHECK_GE(params.fec_rate, 0);
- RTC_DCHECK_LE(params.fec_rate, 255);
+void UlpfecGenerator::SetProtectionParameters(
+ const FecProtectionParams& delta_params,
+ const FecProtectionParams& key_params) {
+ RTC_DCHECK_GE(delta_params.fec_rate, 0);
+ RTC_DCHECK_LE(delta_params.fec_rate, 255);
+ RTC_DCHECK_GE(key_params.fec_rate, 0);
+ RTC_DCHECK_LE(key_params.fec_rate, 255);
// Store the new params and apply them for the next set of FEC packets being
// produced.
- new_params_ = params;
- if (params.fec_rate > kHighProtectionThreshold) {
- min_num_media_packets_ = kMinMediaPackets;
- } else {
- min_num_media_packets_ = 1;
- }
+ rtc::CritScope cs(&crit_);
+ pending_params_.emplace(delta_params, key_params);
}
-int UlpfecGenerator::AddRtpPacketAndGenerateFec(
- const rtc::CopyOnWriteBuffer& data_buffer,
- size_t rtp_header_length) {
+void UlpfecGenerator::AddPacketAndGenerateFec(const RtpPacketToSend& packet) {
+ RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
RTC_DCHECK(generated_fec_packets_.empty());
+
if (media_packets_.empty()) {
- params_ = new_params_;
+ rtc::CritScope cs(&crit_);
+ if (pending_params_) {
+ current_params_ = *pending_params_;
+ pending_params_.reset();
+
+ if (CurrentParams().fec_rate > kHighProtectionThreshold) {
+ min_num_media_packets_ = kMinMediaPackets;
+ } else {
+ min_num_media_packets_ = 1;
+ }
+ }
+
+ keyframe_in_process_ = packet.is_key_frame();
}
+ RTC_DCHECK_EQ(packet.is_key_frame(), keyframe_in_process_);
+
bool complete_frame = false;
- const bool marker_bit = (data_buffer[1] & kRtpMarkerBitMask) ? true : false;
+ const bool marker_bit = packet.Marker();
if (media_packets_.size() < kUlpfecMaxMediaPackets) {
// Our packet masks can only protect up to |kUlpfecMaxMediaPackets| packets.
- std::unique_ptr<ForwardErrorCorrection::Packet> packet(
- new ForwardErrorCorrection::Packet());
- RTC_DCHECK_GE(data_buffer.size(), rtp_header_length);
- packet->data = data_buffer;
- media_packets_.push_back(std::move(packet));
- // Keep track of the RTP header length, so we can copy the RTP header
- // from |packet| to newly generated ULPFEC+RED packets.
- RTC_DCHECK_GE(rtp_header_length, kRtpHeaderSize);
- last_media_packet_rtp_header_length_ = rtp_header_length;
+ auto fec_packet = std::make_unique<ForwardErrorCorrection::Packet>();
+ fec_packet->data = packet.Buffer();
+ media_packets_.push_back(std::move(fec_packet));
+
+ // Keep a copy of the last RTP packet, so we can copy the RTP header
+ // from it when creating newly generated ULPFEC+RED packets.
+ RTC_DCHECK_GE(packet.headers_size(), kRtpHeaderSize);
+ last_media_packet_ = packet;
}
+
if (marker_bit) {
++num_protected_frames_;
complete_frame = true;
}
+
+ auto params = CurrentParams();
+
// Produce FEC over at most |params_.max_fec_frames| frames, or as soon as:
// (1) the excess overhead (actual overhead - requested/target overhead) is
// less than |kMaxExcessOverhead|, and
// (2) at least |min_num_media_packets_| media packets is reached.
if (complete_frame &&
- (num_protected_frames_ == params_.max_fec_frames ||
+ (num_protected_frames_ == params.max_fec_frames ||
(ExcessOverheadBelowMax() && MinimumMediaPacketsReached()))) {
// We are not using Unequal Protection feature of the parity erasure code.
constexpr int kNumImportantPackets = 0;
constexpr bool kUseUnequalProtection = false;
- int ret = fec_->EncodeFec(media_packets_, params_.fec_rate,
- kNumImportantPackets, kUseUnequalProtection,
- params_.fec_mask_type, &generated_fec_packets_);
+ fec_->EncodeFec(media_packets_, params.fec_rate, kNumImportantPackets,
+ kUseUnequalProtection, params.fec_mask_type,
+ &generated_fec_packets_);
if (generated_fec_packets_.empty()) {
ResetState();
}
- return ret;
}
- return 0;
}
bool UlpfecGenerator::ExcessOverheadBelowMax() const {
- return ((Overhead() - params_.fec_rate) < kMaxExcessOverhead);
+ RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
+
+ return ((Overhead() - CurrentParams().fec_rate) < kMaxExcessOverhead);
}
bool UlpfecGenerator::MinimumMediaPacketsReached() const {
+ RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
float average_num_packets_per_frame =
static_cast<float>(media_packets_.size()) / num_protected_frames_;
int num_media_packets = static_cast<int>(media_packets_.size());
@@ -196,61 +188,79 @@
}
}
-bool UlpfecGenerator::FecAvailable() const {
- return !generated_fec_packets_.empty();
-}
-
-size_t UlpfecGenerator::NumAvailableFecPackets() const {
- return generated_fec_packets_.size();
+const FecProtectionParams& UlpfecGenerator::CurrentParams() const {
+ RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
+ return keyframe_in_process_ ? current_params_.keyframe_params
+ : current_params_.delta_params;
}
size_t UlpfecGenerator::MaxPacketOverhead() const {
+ RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
return fec_->MaxPacketOverhead();
}
-std::vector<std::unique_ptr<RedPacket>> UlpfecGenerator::GetUlpfecPacketsAsRed(
- int red_payload_type,
- int ulpfec_payload_type,
- uint16_t first_seq_num) {
- std::vector<std::unique_ptr<RedPacket>> red_packets;
- red_packets.reserve(generated_fec_packets_.size());
- RTC_DCHECK(!media_packets_.empty());
- ForwardErrorCorrection::Packet* last_media_packet =
- media_packets_.back().get();
- uint16_t seq_num = first_seq_num;
+std::vector<std::unique_ptr<RtpPacketToSend>> UlpfecGenerator::GetFecPackets() {
+ RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
+ if (generated_fec_packets_.empty()) {
+ return std::vector<std::unique_ptr<RtpPacketToSend>>();
+ }
+
+ // Wrap FEC packet (including FEC headers) in a RED packet. Since the
+ // FEC packets in |generated_fec_packets_| don't have RTP headers, we
+ // reuse the header from the last media packet.
+ RTC_CHECK(last_media_packet_.has_value());
+ last_media_packet_->SetPayloadSize(0);
+
+ std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets;
+ fec_packets.reserve(generated_fec_packets_.size());
+
+ size_t total_fec_size_bytes = 0;
for (const auto* fec_packet : generated_fec_packets_) {
- // Wrap FEC packet (including FEC headers) in a RED packet. Since the
- // FEC packets in |generated_fec_packets_| don't have RTP headers, we
- // reuse the header from the last media packet.
- RTC_DCHECK_GT(last_media_packet_rtp_header_length_, 0);
- std::unique_ptr<RedPacket> red_packet(
- new RedPacket(last_media_packet_rtp_header_length_ +
- kRedForFecHeaderLength + fec_packet->data.size()));
- red_packet->CreateHeader(last_media_packet->data.data(),
- last_media_packet_rtp_header_length_,
- red_payload_type, ulpfec_payload_type);
- red_packet->SetSeqNum(seq_num++);
- red_packet->ClearMarkerBit();
- red_packet->AssignPayload(fec_packet->data.data(), fec_packet->data.size());
- red_packets.push_back(std::move(red_packet));
+ std::unique_ptr<RtpPacketToSend> red_packet =
+ std::make_unique<RtpPacketToSend>(*last_media_packet_);
+ red_packet->SetPayloadType(red_payload_type_);
+ red_packet->SetMarker(false);
+ uint8_t* payload_buffer = red_packet->SetPayloadSize(
+ kRedForFecHeaderLength + fec_packet->data.size());
+ // Primary RED header with F bit unset.
+ // See https://tools.ietf.org/html/rfc2198#section-3
+ payload_buffer[0] = ulpfec_payload_type_; // RED header.
+ memcpy(&payload_buffer[1], fec_packet->data.data(),
+ fec_packet->data.size());
+ total_fec_size_bytes += red_packet->size();
+ red_packet->set_packet_type(RtpPacketMediaType::kForwardErrorCorrection);
+ red_packet->set_allow_retransmission(false);
+ fec_packets.push_back(std::move(red_packet));
}
ResetState();
- return red_packets;
+ rtc::CritScope cs(&crit_);
+ fec_bitrate_.Update(total_fec_size_bytes, clock_->TimeInMilliseconds());
+
+ return fec_packets;
+}
+
+DataRate UlpfecGenerator::CurrentFecRate() const {
+ rtc::CritScope cs(&crit_);
+ return DataRate::BitsPerSec(
+ fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0));
}
int UlpfecGenerator::Overhead() const {
+ RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
RTC_DCHECK(!media_packets_.empty());
int num_fec_packets =
- fec_->NumFecPackets(media_packets_.size(), params_.fec_rate);
+ fec_->NumFecPackets(media_packets_.size(), CurrentParams().fec_rate);
+
// Return the overhead in Q8.
return (num_fec_packets << 8) / media_packets_.size();
}
void UlpfecGenerator::ResetState() {
+ RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
media_packets_.clear();
- last_media_packet_rtp_header_length_ = 0;
+ last_media_packet_.reset();
generated_fec_packets_.clear();
num_protected_frames_ = 0;
}
diff --git a/modules/rtp_rtcp/source/ulpfec_generator.h b/modules/rtp_rtcp/source/ulpfec_generator.h
index cdfa1ff..6c65f5f 100644
--- a/modules/rtp_rtcp/source/ulpfec_generator.h
+++ b/modules/rtp_rtcp/source/ulpfec_generator.h
@@ -20,63 +20,54 @@
#include "modules/include/module_fec_types.h"
#include "modules/rtp_rtcp/source/forward_error_correction.h"
+#include "modules/rtp_rtcp/source/video_fec_generator.h"
+#include "rtc_base/critical_section.h"
+#include "rtc_base/race_checker.h"
+#include "rtc_base/rate_statistics.h"
namespace webrtc {
class FlexfecSender;
-class RedPacket {
- public:
- explicit RedPacket(size_t length);
- ~RedPacket();
-
- void CreateHeader(const uint8_t* rtp_header,
- size_t header_length,
- int red_payload_type,
- int payload_type);
- void SetSeqNum(int seq_num);
- void AssignPayload(const uint8_t* payload, size_t length);
- void ClearMarkerBit();
- uint8_t* data() const;
- size_t length() const;
-
- private:
- std::unique_ptr<uint8_t[]> data_;
- size_t length_;
- size_t header_length_;
-};
-
-class UlpfecGenerator {
+class UlpfecGenerator : public VideoFecGenerator {
friend class FlexfecSender;
public:
- UlpfecGenerator();
+ UlpfecGenerator(int red_payload_type, int ulpfec_payload_type, Clock* clock);
~UlpfecGenerator();
- void SetFecParameters(const FecProtectionParams& params);
+ FecType GetFecType() const override {
+ return VideoFecGenerator::FecType::kUlpFec;
+ }
+ absl::optional<uint32_t> FecSsrc() override { return absl::nullopt; }
+
+ void SetProtectionParameters(const FecProtectionParams& delta_params,
+ const FecProtectionParams& key_params) override;
// Adds a media packet to the internal buffer. When enough media packets
// have been added, the FEC packets are generated and stored internally.
// These FEC packets are then obtained by calling GetFecPacketsAsRed().
- int AddRtpPacketAndGenerateFec(const rtc::CopyOnWriteBuffer& data_buffer,
- size_t rtp_header_length);
-
- // Returns true if there are generated FEC packets available.
- bool FecAvailable() const;
-
- size_t NumAvailableFecPackets() const;
+ void AddPacketAndGenerateFec(const RtpPacketToSend& packet) override;
// Returns the overhead, per packet, for FEC (and possibly RED).
- size_t MaxPacketOverhead() const;
+ size_t MaxPacketOverhead() const override;
- // Returns generated FEC packets with RED headers added.
- std::vector<std::unique_ptr<RedPacket>> GetUlpfecPacketsAsRed(
- int red_payload_type,
- int ulpfec_payload_type,
- uint16_t first_seq_num);
+ std::vector<std::unique_ptr<RtpPacketToSend>> GetFecPackets() override;
+
+ // Current rate of FEC packets generated, including all RTP-level headers.
+ DataRate CurrentFecRate() const override;
private:
- explicit UlpfecGenerator(std::unique_ptr<ForwardErrorCorrection> fec);
+ struct Params {
+ Params();
+ Params(FecProtectionParams delta_params,
+ FecProtectionParams keyframe_params);
+
+ FecProtectionParams delta_params;
+ FecProtectionParams keyframe_params;
+ };
+
+ UlpfecGenerator(std::unique_ptr<ForwardErrorCorrection> fec, Clock* clock);
// Overhead is defined as relative to the number of media packets, and not
// relative to total number of packets. This definition is inherited from the
@@ -97,16 +88,31 @@
// (e.g. (2k,2m) vs (k,m)) are generally more effective at recovering losses.
bool MinimumMediaPacketsReached() const;
+ const FecProtectionParams& CurrentParams() const;
+
void ResetState();
- std::unique_ptr<ForwardErrorCorrection> fec_;
- ForwardErrorCorrection::PacketList media_packets_;
- size_t last_media_packet_rtp_header_length_;
- std::list<ForwardErrorCorrection::Packet*> generated_fec_packets_;
- int num_protected_frames_;
- int min_num_media_packets_;
- FecProtectionParams params_;
- FecProtectionParams new_params_;
+ const int red_payload_type_;
+ const int ulpfec_payload_type_;
+ Clock* const clock_;
+
+ rtc::RaceChecker race_checker_;
+ const std::unique_ptr<ForwardErrorCorrection> fec_
+ RTC_GUARDED_BY(race_checker_);
+ ForwardErrorCorrection::PacketList media_packets_
+ RTC_GUARDED_BY(race_checker_);
+ absl::optional<RtpPacketToSend> last_media_packet_
+ RTC_GUARDED_BY(race_checker_);
+ std::list<ForwardErrorCorrection::Packet*> generated_fec_packets_
+ RTC_GUARDED_BY(race_checker_);
+ int num_protected_frames_ RTC_GUARDED_BY(race_checker_);
+ int min_num_media_packets_ RTC_GUARDED_BY(race_checker_);
+ Params current_params_ RTC_GUARDED_BY(race_checker_);
+ bool keyframe_in_process_ RTC_GUARDED_BY(race_checker_);
+
+ rtc::CriticalSection crit_;
+ absl::optional<Params> pending_params_ RTC_GUARDED_BY(crit_);
+ RateStatistics fec_bitrate_ RTC_GUARDED_BY(crit_);
};
} // namespace webrtc
diff --git a/modules/rtp_rtcp/source/ulpfec_generator_unittest.cc b/modules/rtp_rtcp/source/ulpfec_generator_unittest.cc
index 8c1c7ea..db005dd 100644
--- a/modules/rtp_rtcp/source/ulpfec_generator_unittest.cc
+++ b/modules/rtp_rtcp/source/ulpfec_generator_unittest.cc
@@ -35,11 +35,8 @@
uint32_t timestamp,
int red_payload_type,
int fec_payload_type,
- RedPacket* packet,
- bool marker_bit) {
- EXPECT_GT(packet->length(), kRtpHeaderSize);
- EXPECT_TRUE(packet->data() != NULL);
- uint8_t* data = packet->data();
+ bool marker_bit,
+ const rtc::CopyOnWriteBuffer& data) {
// Marker bit not set.
EXPECT_EQ(marker_bit ? 0x80 : 0, data[1] & 0x80);
EXPECT_EQ(red_payload_type, data[1] & 0x7F);
@@ -52,8 +49,12 @@
class UlpfecGeneratorTest : public ::testing::Test {
protected:
- UlpfecGeneratorTest() : packet_generator_(kMediaSsrc) {}
+ UlpfecGeneratorTest()
+ : fake_clock_(1),
+ ulpfec_generator_(kRedPayloadType, kFecPayloadType, &fake_clock_),
+ packet_generator_(kMediaSsrc) {}
+ SimulatedClock fake_clock_;
UlpfecGenerator ulpfec_generator_;
AugmentedPacketGenerator packet_generator_;
};
@@ -81,24 +82,22 @@
protected_packets.push_back({21, 0, 55, 0});
protected_packets.push_back({13, 3, 57, 1});
FecProtectionParams params = {117, 3, kFecMaskBursty};
- ulpfec_generator_.SetFecParameters(params);
- uint8_t packet[28] = {0};
+ ulpfec_generator_.SetProtectionParameters(params, params);
for (Packet p : protected_packets) {
- if (p.marker_bit) {
- packet[1] |= 0x80;
+ RtpPacketToSend packet(nullptr);
+ packet.SetMarker(p.marker_bit);
+ packet.AllocateExtension(RTPExtensionType::kRtpExtensionMid,
+ p.header_size - packet.headers_size());
+ packet.SetSequenceNumber(p.seq_num);
+ packet.AllocatePayload(p.payload_size);
+ ulpfec_generator_.AddPacketAndGenerateFec(packet);
+
+ std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
+ ulpfec_generator_.GetFecPackets();
+ if (!p.marker_bit) {
+ EXPECT_TRUE(fec_packets.empty());
} else {
- packet[1] &= ~0x80;
- }
- ByteWriter<uint16_t>::WriteBigEndian(&packet[2], p.seq_num);
- ulpfec_generator_.AddRtpPacketAndGenerateFec(
- rtc::CopyOnWriteBuffer(packet, p.payload_size + p.header_size),
- p.header_size);
- size_t num_fec_packets = ulpfec_generator_.NumAvailableFecPackets();
- if (num_fec_packets > 0) {
- std::vector<std::unique_ptr<RedPacket>> fec_packets =
- ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType,
- kFecPayloadType, 100);
- EXPECT_EQ(num_fec_packets, fec_packets.size());
+ EXPECT_FALSE(fec_packets.empty());
}
}
}
@@ -113,24 +112,28 @@
constexpr size_t kNumPackets = 4;
FecProtectionParams params = {15, 3, kFecMaskRandom};
packet_generator_.NewFrame(kNumPackets);
- ulpfec_generator_.SetFecParameters(params); // Expecting one FEC packet.
+ // Expecting one FEC packet.
+ ulpfec_generator_.SetProtectionParameters(params, params);
uint32_t last_timestamp = 0;
for (size_t i = 0; i < kNumPackets; ++i) {
std::unique_ptr<AugmentedPacket> packet =
packet_generator_.NextPacket(i, 10);
- EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec(packet->data,
- kRtpHeaderSize));
+ RtpPacketToSend rtp_packet(nullptr);
+ EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size()));
+ ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet);
last_timestamp = packet->header.timestamp;
}
- EXPECT_TRUE(ulpfec_generator_.FecAvailable());
- const uint16_t seq_num = packet_generator_.NextPacketSeqNum();
- std::vector<std::unique_ptr<RedPacket>> red_packets =
- ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType,
- seq_num);
- EXPECT_FALSE(ulpfec_generator_.FecAvailable());
- ASSERT_EQ(1u, red_packets.size());
- VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType,
- red_packets.front().get(), false);
+ std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
+ ulpfec_generator_.GetFecPackets();
+ EXPECT_EQ(fec_packets.size(), 1u);
+ uint16_t seq_num = packet_generator_.NextPacketSeqNum();
+ fec_packets[0]->SetSequenceNumber(seq_num);
+ EXPECT_TRUE(ulpfec_generator_.GetFecPackets().empty());
+
+ EXPECT_EQ(fec_packets[0]->headers_size(), kRtpHeaderSize);
+
+ VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType, false,
+ fec_packets[0]->Buffer());
}
TEST_F(UlpfecGeneratorTest, TwoFrameFec) {
@@ -145,27 +148,27 @@
constexpr size_t kNumFrames = 2;
FecProtectionParams params = {15, 3, kFecMaskRandom};
- ulpfec_generator_.SetFecParameters(params); // Expecting one FEC packet.
+ // Expecting one FEC packet.
+ ulpfec_generator_.SetProtectionParameters(params, params);
uint32_t last_timestamp = 0;
for (size_t i = 0; i < kNumFrames; ++i) {
packet_generator_.NewFrame(kNumPackets);
for (size_t j = 0; j < kNumPackets; ++j) {
std::unique_ptr<AugmentedPacket> packet =
packet_generator_.NextPacket(i * kNumPackets + j, 10);
- EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec(
- packet->data, kRtpHeaderSize));
+ RtpPacketToSend rtp_packet(nullptr);
+ EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size()));
+ ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet);
last_timestamp = packet->header.timestamp;
}
}
- EXPECT_TRUE(ulpfec_generator_.FecAvailable());
+ std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
+ ulpfec_generator_.GetFecPackets();
+ EXPECT_EQ(fec_packets.size(), 1u);
const uint16_t seq_num = packet_generator_.NextPacketSeqNum();
- std::vector<std::unique_ptr<RedPacket>> red_packets =
- ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType,
- seq_num);
- EXPECT_FALSE(ulpfec_generator_.FecAvailable());
- ASSERT_EQ(1u, red_packets.size());
- VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType,
- red_packets.front().get(), false);
+ fec_packets[0]->SetSequenceNumber(seq_num);
+ VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType, false,
+ fec_packets[0]->Buffer());
}
TEST_F(UlpfecGeneratorTest, MixedMediaRtpHeaderLengths) {
@@ -174,34 +177,43 @@
// Only one frame required to generate FEC.
FecProtectionParams params = {127, 1, kFecMaskRandom};
- ulpfec_generator_.SetFecParameters(params);
+ ulpfec_generator_.SetProtectionParameters(params, params);
// Fill up internal buffer with media packets with short RTP header length.
packet_generator_.NewFrame(kUlpfecMaxMediaPackets + 1);
for (size_t i = 0; i < kUlpfecMaxMediaPackets; ++i) {
std::unique_ptr<AugmentedPacket> packet =
packet_generator_.NextPacket(i, 10);
- EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec(
- packet->data, kShortRtpHeaderLength));
- EXPECT_FALSE(ulpfec_generator_.FecAvailable());
+ RtpPacketToSend rtp_packet(nullptr);
+ EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size()));
+ EXPECT_EQ(rtp_packet.headers_size(), kShortRtpHeaderLength);
+ ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet);
+ EXPECT_TRUE(ulpfec_generator_.GetFecPackets().empty());
}
// Kick off FEC generation with media packet with long RTP header length.
// Since the internal buffer is full, this packet will not be protected.
std::unique_ptr<AugmentedPacket> packet =
packet_generator_.NextPacket(kUlpfecMaxMediaPackets, 10);
- EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec(
- packet->data, kLongRtpHeaderLength));
- EXPECT_TRUE(ulpfec_generator_.FecAvailable());
+ RtpPacketToSend rtp_packet(nullptr);
+ EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size()));
+ EXPECT_TRUE(rtp_packet.SetPayloadSize(0) != nullptr);
+ const uint32_t csrcs[]{1};
+ rtp_packet.SetCsrcs(csrcs);
+
+ EXPECT_EQ(rtp_packet.headers_size(), kLongRtpHeaderLength);
+
+ ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet);
+ std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
+ ulpfec_generator_.GetFecPackets();
+ EXPECT_FALSE(fec_packets.empty());
// Ensure that the RED header is placed correctly, i.e. the correct
// RTP header length was used in the RED packet creation.
- const uint16_t seq_num = packet_generator_.NextPacketSeqNum();
- std::vector<std::unique_ptr<RedPacket>> red_packets =
- ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType,
- seq_num);
- for (const auto& red_packet : red_packets) {
- EXPECT_EQ(kFecPayloadType, red_packet->data()[kShortRtpHeaderLength]);
+ uint16_t seq_num = packet_generator_.NextPacketSeqNum();
+ for (const auto& fec_packet : fec_packets) {
+ fec_packet->SetSequenceNumber(seq_num++);
+ EXPECT_EQ(kFecPayloadType, fec_packet->data()[kShortRtpHeaderLength]);
}
}
diff --git a/modules/rtp_rtcp/source/video_fec_generator.h b/modules/rtp_rtcp/source/video_fec_generator.h
new file mode 100644
index 0000000..3731449
--- /dev/null
+++ b/modules/rtp_rtcp/source/video_fec_generator.h
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_RTP_RTCP_SOURCE_VIDEO_FEC_GENERATOR_H_
+#define MODULES_RTP_RTCP_SOURCE_VIDEO_FEC_GENERATOR_H_
+
+#include <memory>
+#include <vector>
+
+#include "api/units/data_rate.h"
+#include "modules/include/module_fec_types.h"
+#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
+
+namespace webrtc {
+
+class VideoFecGenerator {
+ public:
+ VideoFecGenerator() = default;
+ virtual ~VideoFecGenerator() = default;
+
+ enum class FecType { kFlexFec, kUlpFec };
+ virtual FecType GetFecType() const = 0;
+ // Returns the SSRC used for FEC packets (i.e. FlexFec SSRC).
+ virtual absl::optional<uint32_t> FecSsrc() = 0;
+ // Returns the overhead, in bytes per packet, for FEC (and possibly RED).
+ virtual size_t MaxPacketOverhead() const = 0;
+ // Current rate of FEC packets generated, including all RTP-level headers.
+ virtual DataRate CurrentFecRate() const = 0;
+ // Set FEC rates, max frames before FEC is sent, and type of FEC masks.
+ virtual void SetProtectionParameters(
+ const FecProtectionParams& delta_params,
+ const FecProtectionParams& key_params) = 0;
+ // Called on new media packet to be protected. The generator may choose
+ // to generate FEC packets at this time, if so they will be stored in an
+ // internal buffer.
+ virtual void AddPacketAndGenerateFec(const RtpPacketToSend& packet) = 0;
+ // Get (and remove) and FEC packets pending in the generator. These packets
+ // will lack sequence numbers, that needs to be set externally.
+ // TODO(bugs.webrtc.org/11340): Actually FlexFec sets seq#, fix that!
+ virtual std::vector<std::unique_ptr<RtpPacketToSend>> GetFecPackets() = 0;
+};
+
+} // namespace webrtc
+#endif // MODULES_RTP_RTCP_SOURCE_VIDEO_FEC_GENERATOR_H_
diff --git a/test/fuzzers/BUILD.gn b/test/fuzzers/BUILD.gn
index 4caac34..123e548 100644
--- a/test/fuzzers/BUILD.gn
+++ b/test/fuzzers/BUILD.gn
@@ -149,6 +149,7 @@
"../../modules/rtp_rtcp:rtp_rtcp_format",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
+ "../../system_wrappers",
]
}
diff --git a/test/fuzzers/flexfec_sender_fuzzer.cc b/test/fuzzers/flexfec_sender_fuzzer.cc
index 4882f7d..8ddd1c0 100644
--- a/test/fuzzers/flexfec_sender_fuzzer.cc
+++ b/test/fuzzers/flexfec_sender_fuzzer.cc
@@ -41,7 +41,7 @@
FecProtectionParams params = {
data[i++], static_cast<int>(data[i++] % 100),
data[i++] <= 127 ? kFecMaskRandom : kFecMaskBursty};
- sender.SetFecParameters(params);
+ sender.SetProtectionParameters(params, params);
uint16_t seq_num = data[i++];
while (i + 1 < size) {
@@ -59,11 +59,8 @@
RtpPacketToSend rtp_packet(nullptr);
if (!rtp_packet.Parse(packet.get(), kRtpHeaderSize + payload_size))
break;
- sender.AddRtpPacketAndGenerateFec(rtp_packet);
- if (sender.FecAvailable()) {
- std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
- sender.GetFecPackets();
- }
+ sender.AddPacketAndGenerateFec(rtp_packet);
+ sender.GetFecPackets();
}
}
diff --git a/test/fuzzers/ulpfec_generator_fuzzer.cc b/test/fuzzers/ulpfec_generator_fuzzer.cc
index 306f7a0..9426ef0 100644
--- a/test/fuzzers/ulpfec_generator_fuzzer.cc
+++ b/test/fuzzers/ulpfec_generator_fuzzer.cc
@@ -16,6 +16,7 @@
#include "modules/rtp_rtcp/source/ulpfec_generator.h"
#include "rtc_base/checks.h"
#include "rtc_base/copy_on_write_buffer.h"
+#include "system_wrappers/include/clock.h"
namespace webrtc {
@@ -25,13 +26,14 @@
} // namespace
void FuzzOneInput(const uint8_t* data, size_t size) {
- UlpfecGenerator generator;
+ SimulatedClock clock(1);
+ UlpfecGenerator generator(kRedPayloadType, kFecPayloadType, &clock);
size_t i = 0;
if (size < 4)
return;
FecProtectionParams params = {
data[i++] % 128, static_cast<int>(data[i++] % 10), kFecMaskBursty};
- generator.SetFecParameters(params);
+ generator.SetProtectionParameters(params, params);
uint16_t seq_num = data[i++];
uint16_t prev_seq_num = 0;
while (i + 3 < size) {
@@ -41,6 +43,9 @@
break;
rtc::CopyOnWriteBuffer packet(&data[i], payload_size + rtp_header_length);
packet.EnsureCapacity(IP_PACKET_SIZE);
+ // Write a valid parsable header (version = 2, no padding, no extensions,
+ // no CSRCs).
+ ByteWriter<uint8_t>::WriteBigEndian(&packet[0], 2 << 6);
// Make sure sequence numbers are increasing.
ByteWriter<uint16_t>::WriteBigEndian(&packet[2], seq_num++);
i += payload_size + rtp_header_length;
@@ -51,16 +56,15 @@
// number became out of order.
if (protect && IsNewerSequenceNumber(seq_num, prev_seq_num) &&
seq_num < prev_seq_num + kUlpfecMaxMediaPackets) {
- generator.AddRtpPacketAndGenerateFec(packet, rtp_header_length);
+ RtpPacketToSend rtp_packet(nullptr);
+ // Check that we actually have a parsable packet, we want to fuzz FEC
+ // logic, not RTP header parsing.
+ RTC_CHECK(rtp_packet.Parse(packet));
+ generator.AddPacketAndGenerateFec(rtp_packet);
prev_seq_num = seq_num;
}
- const size_t num_fec_packets = generator.NumAvailableFecPackets();
- if (num_fec_packets > 0) {
- std::vector<std::unique_ptr<RedPacket>> fec_packets =
- generator.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType,
- 100);
- RTC_CHECK_EQ(num_fec_packets, fec_packets.size());
- }
+
+ generator.GetFecPackets();
}
}
} // namespace webrtc