Frame marking RTP header extension (PART 1: implement extension)
Bug: webrtc:7765
Change-Id: I23896d121afd6be4bce5ff4deaf736149efebcdb
Reviewed-on: https://webrtc-review.googlesource.com/85200
Commit-Queue: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24695}diff --git a/AUTHORS b/AUTHORS
index 05b6bbe..f03daca 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -90,6 +90,7 @@
The Chromium Authors <*@chromium.org>
The WebRTC Authors <*@webrtc.org>
Videxio AS <*@videxio.com>
+Vidyo, Inc. <*@vidyo.com>
Vonage Holdings Corp. <*@vonage.com>
Wire Swiss GmbH <*@wire.com>
Miguel Paris <mparisdiaz@gmail.com>
diff --git a/api/rtp_headers.cc b/api/rtp_headers.cc
index a0b1a15..da7f1ea 100644
--- a/api/rtp_headers.cc
+++ b/api/rtp_headers.cc
@@ -34,7 +34,9 @@
videoRotation(kVideoRotation_0),
hasVideoContentType(false),
videoContentType(VideoContentType::UNSPECIFIED),
- has_video_timing(false) {}
+ has_video_timing(false),
+ has_frame_marking(false),
+ frame_marking({false, false, false, false, false, 0xFF, 0, 0}) {}
RTPHeaderExtension::RTPHeaderExtension(const RTPHeaderExtension& other) =
default;
diff --git a/api/rtp_headers.h b/api/rtp_headers.h
index c762534..41f3988 100644
--- a/api/rtp_headers.h
+++ b/api/rtp_headers.h
@@ -19,6 +19,7 @@
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/video/video_content_type.h"
+#include "api/video/video_frame_marking.h"
#include "api/video/video_rotation.h"
#include "api/video/video_timing.h"
@@ -116,6 +117,9 @@
bool has_video_timing;
VideoSendTiming video_timing;
+ bool has_frame_marking;
+ FrameMarking frame_marking;
+
PlayoutDelay playout_delay = {-1, -1};
// For identification of a stream when ssrc is not signaled. See
diff --git a/api/rtpparameters.cc b/api/rtpparameters.cc
index 62ca3fc..9beae82 100644
--- a/api/rtpparameters.cc
+++ b/api/rtpparameters.cc
@@ -129,6 +129,10 @@
const char RtpExtension::kMidUri[] = "urn:ietf:params:rtp-hdrext:sdes:mid";
const int RtpExtension::kMidDefaultId = 9;
+const char RtpExtension::kFrameMarkingUri[] =
+ "http://tools.ietf.org/html/draft-ietf-avtext-framemarking-07";
+const int RtpExtension::kFrameMarkingDefaultId = 10;
+
const char RtpExtension::kEncryptHeaderExtensionsUri[] =
"urn:ietf:params:rtp-hdrext:encrypt";
@@ -149,7 +153,8 @@
uri == webrtc::RtpExtension::kPlayoutDelayUri ||
uri == webrtc::RtpExtension::kVideoContentTypeUri ||
uri == webrtc::RtpExtension::kVideoTimingUri ||
- uri == webrtc::RtpExtension::kMidUri;
+ uri == webrtc::RtpExtension::kMidUri ||
+ uri == webrtc::RtpExtension::kFrameMarkingUri;
}
bool RtpExtension::IsEncryptionSupported(const std::string& uri) {
diff --git a/api/rtpparameters.h b/api/rtpparameters.h
index 9a29c08..aa9b09e 100644
--- a/api/rtpparameters.h
+++ b/api/rtpparameters.h
@@ -282,6 +282,10 @@
static const char kVideoTimingUri[];
static const int kVideoTimingDefaultId;
+ // Header extension for video frame marking.
+ static const char kFrameMarkingUri[];
+ static const int kFrameMarkingDefaultId;
+
// Header extension for transport sequence number, see url for details:
// http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions
static const char kTransportSequenceNumberUri[];
diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn
index dfc4f27..d041260 100644
--- a/api/video/BUILD.gn
+++ b/api/video/BUILD.gn
@@ -19,6 +19,7 @@
"video_frame.h",
"video_frame_buffer.cc",
"video_frame_buffer.h",
+ "video_frame_marking.h",
"video_rotation.h",
"video_sink_interface.h",
"video_source_interface.cc",
diff --git a/api/video/video_frame_marking.h b/api/video/video_frame_marking.h
new file mode 100644
index 0000000..2a34852
--- /dev/null
+++ b/api/video/video_frame_marking.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_VIDEO_VIDEO_FRAME_MARKING_H_
+#define API_VIDEO_VIDEO_FRAME_MARKING_H_
+
+namespace webrtc {
+
+struct FrameMarking {
+ bool start_of_frame;
+ bool end_of_frame;
+ bool independent_frame;
+ bool discardable_frame;
+ bool base_layer_sync;
+ uint8_t temporal_id;
+ uint8_t layer_id;
+ uint8_t tl0_pic_idx;
+};
+
+} // namespace webrtc
+
+#endif // API_VIDEO_VIDEO_FRAME_MARKING_H_
diff --git a/logging/rtc_event_log/rtc_event_log2text.cc b/logging/rtc_event_log/rtc_event_log2text.cc
index f08e6ec..67cb6ec 100644
--- a/logging/rtc_event_log/rtc_event_log2text.cc
+++ b/logging/rtc_event_log/rtc_event_log2text.cc
@@ -157,6 +157,8 @@
webrtc::RtpExtension::kVideoContentTypeDefaultId);
default_map.Register<webrtc::VideoTimingExtension>(
webrtc::RtpExtension::kVideoTimingDefaultId);
+ default_map.Register<webrtc::FrameMarkingExtension>(
+ webrtc::RtpExtension::kFrameMarkingDefaultId);
default_map.Register<webrtc::TransportSequenceNumber>(
webrtc::RtpExtension::kTransportSequenceNumberDefaultId);
default_map.Register<webrtc::PlayoutDelayLimits>(
diff --git a/logging/rtc_event_log/rtc_event_log_parser_new.cc b/logging/rtc_event_log/rtc_event_log_parser_new.cc
index f905a04..bb5b1b2 100644
--- a/logging/rtc_event_log/rtc_event_log_parser_new.cc
+++ b/logging/rtc_event_log/rtc_event_log_parser_new.cc
@@ -228,6 +228,8 @@
webrtc::RtpExtension::kVideoContentTypeDefaultId);
default_map.Register<VideoTimingExtension>(
webrtc::RtpExtension::kVideoTimingDefaultId);
+ default_map.Register<FrameMarkingExtension>(
+ webrtc::RtpExtension::kFrameMarkingDefaultId);
default_map.Register<TransportSequenceNumber>(
webrtc::RtpExtension::kTransportSequenceNumberDefaultId);
default_map.Register<PlayoutDelayLimits>(
diff --git a/media/engine/webrtcvideoengine.cc b/media/engine/webrtcvideoengine.cc
index 6917384..18919bc 100644
--- a/media/engine/webrtcvideoengine.cc
+++ b/media/engine/webrtcvideoengine.cc
@@ -509,6 +509,9 @@
capabilities.header_extensions.push_back(
webrtc::RtpExtension(webrtc::RtpExtension::kVideoTimingUri,
webrtc::RtpExtension::kVideoTimingDefaultId));
+ capabilities.header_extensions.push_back(
+ webrtc::RtpExtension(webrtc::RtpExtension::kFrameMarkingUri,
+ webrtc::RtpExtension::kFrameMarkingDefaultId));
// TODO(bugs.webrtc.org/4050): Add MID header extension as capability once MID
// demuxing is completed.
// capabilities.header_extensions.push_back(webrtc::RtpExtension(
diff --git a/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/modules/rtp_rtcp/include/rtp_rtcp_defines.h
index 34c7428..51e6337 100644
--- a/modules/rtp_rtcp/include/rtp_rtcp_defines.h
+++ b/modules/rtp_rtcp/include/rtp_rtcp_defines.h
@@ -103,6 +103,7 @@
kRtpExtensionPlayoutDelay,
kRtpExtensionVideoContentType,
kRtpExtensionVideoTiming,
+ kRtpExtensionFrameMarking,
kRtpExtensionRtpStreamId,
kRtpExtensionRepairedRtpStreamId,
kRtpExtensionMid,
diff --git a/modules/rtp_rtcp/source/rtp_header_extension_map.cc b/modules/rtp_rtcp/source/rtp_header_extension_map.cc
index 9fa66c0..23ccd0e 100644
--- a/modules/rtp_rtcp/source/rtp_header_extension_map.cc
+++ b/modules/rtp_rtcp/source/rtp_header_extension_map.cc
@@ -38,6 +38,7 @@
CreateExtensionInfo<PlayoutDelayLimits>(),
CreateExtensionInfo<VideoContentTypeExtension>(),
CreateExtensionInfo<VideoTimingExtension>(),
+ CreateExtensionInfo<FrameMarkingExtension>(),
CreateExtensionInfo<RtpStreamId>(),
CreateExtensionInfo<RepairedRtpStreamId>(),
CreateExtensionInfo<RtpMid>(),
diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.cc b/modules/rtp_rtcp/source/rtp_header_extensions.cc
index 2dba4d7..e9e4d50 100644
--- a/modules/rtp_rtcp/source/rtp_header_extensions.cc
+++ b/modules/rtp_rtcp/source/rtp_header_extensions.cc
@@ -350,6 +350,86 @@
return true;
}
+// Frame Marking.
+//
+// Meta-information about an RTP stream outside the encrypted media payload,
+// useful for an RTP switch to do codec-agnostic selective forwarding
+// without decrypting the payload.
+//
+// For non-scalable streams:
+// 0 1
+// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// | ID | L = 0 |S|E|I|D|0 0 0 0|
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//
+// For scalable streams:
+// 0 1 2 3
+// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// | ID | L = 2 |S|E|I|D|B| TID | LID | TL0PICIDX |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+constexpr RTPExtensionType FrameMarkingExtension::kId;
+constexpr const char FrameMarkingExtension::kUri[];
+
+bool FrameMarkingExtension::IsScalable(uint8_t temporal_id, uint8_t layer_id) {
+ return temporal_id != kNoTemporalIdx || layer_id != kNoSpatialIdx;
+}
+
+bool FrameMarkingExtension::Parse(rtc::ArrayView<const uint8_t> data,
+ FrameMarking* frame_marking) {
+ RTC_DCHECK(frame_marking);
+
+ if (data.size() != 1 && data.size() != 3)
+ return false;
+
+ frame_marking->start_of_frame = (data[0] & 0x80) != 0;
+ frame_marking->end_of_frame = (data[0] & 0x40) != 0;
+ frame_marking->independent_frame = (data[0] & 0x20) != 0;
+ frame_marking->discardable_frame = (data[0] & 0x10) != 0;
+
+ if (data.size() == 3) {
+ frame_marking->base_layer_sync = (data[0] & 0x08) != 0;
+ frame_marking->temporal_id = data[0] & 0x7;
+ frame_marking->layer_id = data[1];
+ frame_marking->tl0_pic_idx = data[2];
+ } else {
+ // non-scalable
+ frame_marking->base_layer_sync = false;
+ frame_marking->temporal_id = kNoTemporalIdx;
+ frame_marking->layer_id = kNoSpatialIdx;
+ frame_marking->tl0_pic_idx = 0;
+ }
+ return true;
+}
+
+size_t FrameMarkingExtension::ValueSize(const FrameMarking& frame_marking) {
+ if (IsScalable(frame_marking.temporal_id, frame_marking.layer_id))
+ return 3;
+ else
+ return 1;
+}
+
+bool FrameMarkingExtension::Write(rtc::ArrayView<uint8_t> data,
+ const FrameMarking& frame_marking) {
+ RTC_DCHECK_GE(data.size(), 1);
+ RTC_CHECK_LE(frame_marking.temporal_id, 0x07);
+ data[0] = frame_marking.start_of_frame ? 0x80 : 0x00;
+ data[0] |= frame_marking.end_of_frame ? 0x40 : 0x00;
+ data[0] |= frame_marking.independent_frame ? 0x20 : 0x00;
+ data[0] |= frame_marking.discardable_frame ? 0x10 : 0x00;
+
+ if (IsScalable(frame_marking.temporal_id, frame_marking.layer_id)) {
+ RTC_DCHECK_EQ(data.size(), 3);
+ data[0] |= frame_marking.base_layer_sync ? 0x08 : 0x00;
+ data[0] |= frame_marking.temporal_id & 0x07;
+ data[1] = frame_marking.layer_id;
+ data[2] = frame_marking.tl0_pic_idx;
+ }
+ return true;
+}
+
bool BaseRtpStringExtension::Parse(rtc::ArrayView<const uint8_t> data,
StringRtpHeaderExtension* str) {
if (data.empty() || data[0] == 0) // Valid string extension can't be empty.
diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.h b/modules/rtp_rtcp/source/rtp_header_extensions.h
index 4e1afc1..84e9831 100644
--- a/modules/rtp_rtcp/source/rtp_header_extensions.h
+++ b/modules/rtp_rtcp/source/rtp_header_extensions.h
@@ -153,6 +153,22 @@
uint8_t idx);
};
+class FrameMarkingExtension {
+ public:
+ static constexpr RTPExtensionType kId = kRtpExtensionFrameMarking;
+ static constexpr const char kUri[] =
+ "http://tools.ietf.org/html/draft-ietf-avtext-framemarking-07";
+
+ static bool Parse(rtc::ArrayView<const uint8_t> data,
+ FrameMarking* frame_marking);
+ static size_t ValueSize(const FrameMarking& frame_marking);
+ static bool Write(rtc::ArrayView<uint8_t> data,
+ const FrameMarking& frame_marking);
+
+ private:
+ static bool IsScalable(uint8_t temporal_id, uint8_t layer_id);
+};
+
// Base extension class for RTP header extensions which are strings.
// Subclasses must defined kId and kUri static constexpr members.
class BaseRtpStringExtension {
diff --git a/modules/rtp_rtcp/source/rtp_packet_received.cc b/modules/rtp_rtcp/source/rtp_packet_received.cc
index 6acc9b5..c8deb99 100644
--- a/modules/rtp_rtcp/source/rtp_packet_received.cc
+++ b/modules/rtp_rtcp/source/rtp_packet_received.cc
@@ -61,6 +61,8 @@
&header->extension.videoContentType);
header->extension.has_video_timing =
GetExtension<VideoTimingExtension>(&header->extension.video_timing);
+ header->extension.has_frame_marking =
+ GetExtension<FrameMarkingExtension>(&header->extension.frame_marking);
GetExtension<RtpStreamId>(&header->extension.stream_id);
GetExtension<RepairedRtpStreamId>(&header->extension.repaired_stream_id);
GetExtension<RtpMid>(&header->extension.mid);
diff --git a/modules/rtp_rtcp/source/rtp_receiver_video.cc b/modules/rtp_rtcp/source/rtp_receiver_video.cc
index 1101bec..3c9b9e5 100644
--- a/modules/rtp_rtcp/source/rtp_receiver_video.cc
+++ b/modules/rtp_rtcp/source/rtp_receiver_video.cc
@@ -74,6 +74,7 @@
rtp_header->video_header().rotation = kVideoRotation_0;
rtp_header->video_header().content_type = VideoContentType::UNSPECIFIED;
rtp_header->video_header().video_timing.flags = VideoSendTiming::kInvalid;
+ rtp_header->video_header().frame_marking.temporal_id = kNoTemporalIdx;
// Retrieve the video rotation information.
if (rtp_header->header.extension.hasVideoRotation) {
@@ -94,6 +95,11 @@
rtp_header->video_header().playout_delay =
rtp_header->header.extension.playout_delay;
+ if (rtp_header->header.extension.has_frame_marking) {
+ rtp_header->video_header().frame_marking =
+ rtp_header->header.extension.frame_marking;
+ }
+
return data_callback_->OnReceivedPayloadData(parsed_payload.payload,
parsed_payload.payload_length,
rtp_header) == 0
diff --git a/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/modules/rtp_rtcp/source/rtp_rtcp_impl.cc
index fe8dbf3..20ae519 100644
--- a/modules/rtp_rtcp/source/rtp_rtcp_impl.cc
+++ b/modules/rtp_rtcp/source/rtp_rtcp_impl.cc
@@ -50,6 +50,8 @@
return kRtpExtensionVideoContentType;
if (extension == RtpExtension::kVideoTimingUri)
return kRtpExtensionVideoTiming;
+ if (extension == RtpExtension::kFrameMarkingUri)
+ return kRtpExtensionFrameMarking;
if (extension == RtpExtension::kMidUri)
return kRtpExtensionMid;
RTC_NOTREACHED() << "Looking up unsupported RTP extension.";
diff --git a/modules/rtp_rtcp/source/rtp_utility.cc b/modules/rtp_rtcp/source/rtp_utility.cc
index d150ce2..5f5b3dc 100644
--- a/modules/rtp_rtcp/source/rtp_utility.cc
+++ b/modules/rtp_rtcp/source/rtp_utility.cc
@@ -236,6 +236,10 @@
header->extension.has_video_timing = false;
header->extension.video_timing = {0u, 0u, 0u, 0u, 0u, 0u, false};
+ header->extension.has_frame_marking = false;
+ header->extension.frame_marking = {false, false, false, false, false,
+ kNoTemporalIdx, 0, 0};
+
if (X) {
/* RTP header extension, RFC 3550.
0 1 2 3
@@ -454,6 +458,15 @@
&header->extension.video_timing);
break;
}
+ case kRtpExtensionFrameMarking: {
+ if (!FrameMarkingExtension::Parse(rtc::MakeArrayView(ptr, len + 1),
+ &header->extension.frame_marking)) {
+ RTC_LOG(LS_WARNING) << "Incorrect frame marking len: " << len;
+ return;
+ }
+ header->extension.has_frame_marking = true;
+ break;
+ }
case kRtpExtensionRtpStreamId: {
header->extension.stream_id.Set(rtc::MakeArrayView(ptr, len + 1));
break;
diff --git a/modules/rtp_rtcp/source/rtp_video_header.h b/modules/rtp_rtcp/source/rtp_video_header.h
index 6c7a150..68b4884 100644
--- a/modules/rtp_rtcp/source/rtp_video_header.h
+++ b/modules/rtp_rtcp/source/rtp_video_header.h
@@ -13,6 +13,7 @@
#include "absl/container/inlined_vector.h"
#include "absl/types/variant.h"
#include "api/video/video_content_type.h"
+#include "api/video/video_frame_marking.h"
#include "api/video/video_rotation.h"
#include "api/video/video_timing.h"
#include "common_types.h" // NOLINT(build/include)
@@ -56,6 +57,7 @@
PlayoutDelay playout_delay;
VideoSendTiming video_timing;
+ FrameMarking frame_marking;
RTPVideoTypeHeader video_type_header;
};
diff --git a/modules/video_coding/frame_object.cc b/modules/video_coding/frame_object.cc
index 0965d7a..4f7724d 100644
--- a/modules/video_coding/frame_object.cc
+++ b/modules/video_coding/frame_object.cc
@@ -159,5 +159,13 @@
return packet->video_header;
}
+absl::optional<FrameMarking> RtpFrameObject::GetFrameMarking() const {
+ rtc::CritScope lock(&packet_buffer_->crit_);
+ VCMPacket* packet = packet_buffer_->GetPacket(first_seq_num_);
+ if (!packet)
+ return absl::nullopt;
+ return packet->video_header.frame_marking;
+}
+
} // namespace video_coding
} // namespace webrtc
diff --git a/modules/video_coding/frame_object.h b/modules/video_coding/frame_object.h
index 08c6404..5fb5193 100644
--- a/modules/video_coding/frame_object.h
+++ b/modules/video_coding/frame_object.h
@@ -41,6 +41,7 @@
int64_t RenderTime() const override;
bool delayed_by_retransmission() const override;
absl::optional<RTPVideoHeader> GetRtpVideoHeader() const;
+ absl::optional<FrameMarking> GetFrameMarking() const;
private:
rtc::scoped_refptr<PacketBuffer> packet_buffer_;
diff --git a/test/fuzzers/rtp_packet_fuzzer.cc b/test/fuzzers/rtp_packet_fuzzer.cc
index 0e35c8e..ef9bf26 100644
--- a/test/fuzzers/rtp_packet_fuzzer.cc
+++ b/test/fuzzers/rtp_packet_fuzzer.cc
@@ -95,6 +95,10 @@
VideoSendTiming timing;
packet.GetExtension<VideoTimingExtension>(&timing);
break;
+ case kRtpExtensionFrameMarking:
+ FrameMarking frame_marking;
+ packet.GetExtension<FrameMarkingExtension>(&frame_marking);
+ break;
case kRtpExtensionRtpStreamId: {
std::string rsid;
packet.GetExtension<RtpStreamId>(&rsid);