Add helper to calculate frame dependencies based on encoder buffer usage
Bug: webrtc:10342
Change-Id: I1d856d060c2defcd10310f0d8639ce8a9554fff3
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/168194
Commit-Queue: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30458}
diff --git a/common_video/generic_frame_descriptor/generic_frame_info.h b/common_video/generic_frame_descriptor/generic_frame_info.h
index ce3ee6c..b602ee0 100644
--- a/common_video/generic_frame_descriptor/generic_frame_info.h
+++ b/common_video/generic_frame_descriptor/generic_frame_info.h
@@ -22,7 +22,7 @@
// Describes how a certain encoder buffer was used when encoding a frame.
struct CodecBufferUsage {
- CodecBufferUsage(int id, bool referenced, bool updated)
+ constexpr CodecBufferUsage(int id, bool referenced, bool updated)
: id(id), referenced(referenced), updated(updated) {}
int id = 0;
diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index 22bb142..4ae65df 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -34,6 +34,26 @@
]
}
+rtc_library("frame_dependencies_calculator") {
+ sources = [
+ "frame_dependencies_calculator.cc",
+ "frame_dependencies_calculator.h",
+ ]
+
+ deps = [
+ "../../api:array_view",
+ "../../api/video:video_frame_type",
+ "../../common_video/generic_frame_descriptor",
+ "../../rtc_base:checks",
+ "../../rtc_base:logging",
+ "../../rtc_base:macromagic",
+ "../../rtc_base/synchronization:sequence_checker",
+ "//third_party/abseil-cpp/absl/algorithm:container",
+ "//third_party/abseil-cpp/absl/container:inlined_vector",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+}
+
rtc_library("nack_module") {
visibility = [ "*" ]
sources = [
@@ -827,6 +847,7 @@
"decoding_state_unittest.cc",
"fec_controller_unittest.cc",
"frame_buffer2_unittest.cc",
+ "frame_dependencies_calculator_unittest.cc",
"generic_decoder_unittest.cc",
"h264_sprop_parameter_sets_unittest.cc",
"h264_sps_pps_tracker_unittest.cc",
@@ -864,6 +885,7 @@
deps = [
":codec_globals_headers",
":encoded_frame",
+ ":frame_dependencies_calculator",
":nack_module",
":simulcast_test_fixture_impl",
":video_codec_interface",
@@ -897,10 +919,12 @@
"../../api/video:video_bitrate_allocator_factory",
"../../api/video:video_frame",
"../../api/video:video_frame_i420",
+ "../../api/video:video_frame_type",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../api/video_codecs:vp8_temporal_layers_factory",
"../../common_video",
+ "../../common_video/generic_frame_descriptor",
"../../common_video/test:utilities",
"../../media:rtc_media_base",
"../../rtc_base",
diff --git a/modules/video_coding/frame_dependencies_calculator.cc b/modules/video_coding/frame_dependencies_calculator.cc
new file mode 100644
index 0000000..c3042d9
--- /dev/null
+++ b/modules/video_coding/frame_dependencies_calculator.cc
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/frame_dependencies_calculator.h"
+
+#include <stdint.h>
+
+#include <iterator>
+#include <set>
+
+#include "absl/algorithm/container.h"
+#include "absl/container/inlined_vector.h"
+#include "api/array_view.h"
+#include "api/video/video_frame_type.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/synchronization/sequence_checker.h"
+
+namespace webrtc {
+
+absl::InlinedVector<int64_t, 5> FrameDependenciesCalculator::FromBuffersUsage(
+ VideoFrameType frame_type,
+ int64_t frame_id,
+ rtc::ArrayView<const CodecBufferUsage> buffers_usage) {
+ RTC_DCHECK_RUN_ON(&checker_);
+
+ absl::InlinedVector<int64_t, 5> dependencies;
+ RTC_DCHECK_GT(buffers_usage.size(), 0);
+ for (const CodecBufferUsage& buffer_usage : buffers_usage) {
+ RTC_CHECK_GE(buffer_usage.id, 0);
+ if (buffers_.size() <= static_cast<size_t>(buffer_usage.id)) {
+ buffers_.resize(buffer_usage.id + 1);
+ }
+ }
+ std::set<int64_t> direct_depenendencies;
+ std::set<int64_t> indirect_depenendencies;
+ if (frame_type == VideoFrameType::kVideoFrameDelta) {
+ for (const CodecBufferUsage& buffer_usage : buffers_usage) {
+ if (!buffer_usage.referenced) {
+ continue;
+ }
+ const BufferUsage& buffer = buffers_[buffer_usage.id];
+ if (buffer.frame_id == absl::nullopt) {
+ RTC_LOG(LS_ERROR) << "Odd configuration: frame " << frame_id
+ << " references buffer #" << buffer_usage.id
+ << " that was never updated.";
+ continue;
+ }
+ direct_depenendencies.insert(*buffer.frame_id);
+ indirect_depenendencies.insert(buffer.dependencies.begin(),
+ buffer.dependencies.end());
+ }
+ // Reduce references: if frame #3 depends on frame #2 and #1, and frame #2
+ // depends on frame #1, then frame #3 needs to depend just on frame #2.
+ // Though this set diff removes only 1 level of indirection, it seems
+ // enough for all currently used structures.
+ absl::c_set_difference(direct_depenendencies, indirect_depenendencies,
+ std::back_inserter(dependencies));
+ }
+
+ // Update buffers.
+ for (const CodecBufferUsage& buffer_usage : buffers_usage) {
+ if (!buffer_usage.updated) {
+ continue;
+ }
+ BufferUsage& buffer = buffers_[buffer_usage.id];
+ buffer.frame_id = frame_id;
+ buffer.dependencies.assign(direct_depenendencies.begin(),
+ direct_depenendencies.end());
+ }
+
+ return dependencies;
+}
+
+} // namespace webrtc
diff --git a/modules/video_coding/frame_dependencies_calculator.h b/modules/video_coding/frame_dependencies_calculator.h
new file mode 100644
index 0000000..f723d0f
--- /dev/null
+++ b/modules/video_coding/frame_dependencies_calculator.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_FRAME_DEPENDENCIES_CALCULATOR_H_
+#define MODULES_VIDEO_CODING_FRAME_DEPENDENCIES_CALCULATOR_H_
+
+#include <stdint.h>
+
+#include <vector>
+
+#include "absl/container/inlined_vector.h"
+#include "absl/types/optional.h"
+#include "api/array_view.h"
+#include "api/video/video_frame_type.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "rtc_base/synchronization/sequence_checker.h"
+#include "rtc_base/thread_annotations.h"
+
+namespace webrtc {
+
+class FrameDependenciesCalculator {
+ public:
+ FrameDependenciesCalculator() = default;
+ FrameDependenciesCalculator(FrameDependenciesCalculator&&) = default;
+ FrameDependenciesCalculator& operator=(FrameDependenciesCalculator&&) =
+ default;
+
+ // Calculates frame dependencies based on previous encoder buffer usage.
+ absl::InlinedVector<int64_t, 5> FromBuffersUsage(
+ VideoFrameType frame_type,
+ int64_t frame_id,
+ rtc::ArrayView<const CodecBufferUsage> buffers_usage);
+
+ private:
+ struct BufferUsage {
+ absl::optional<int64_t> frame_id;
+ absl::InlinedVector<int64_t, 4> dependencies;
+ };
+
+ SequenceChecker checker_;
+ absl::InlinedVector<BufferUsage, 4> buffers_ RTC_GUARDED_BY(checker_);
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_FRAME_DEPENDENCIES_CALCULATOR_H_
diff --git a/modules/video_coding/frame_dependencies_calculator_unittest.cc b/modules/video_coding/frame_dependencies_calculator_unittest.cc
new file mode 100644
index 0000000..81f774b
--- /dev/null
+++ b/modules/video_coding/frame_dependencies_calculator_unittest.cc
@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/frame_dependencies_calculator.h"
+
+#include "api/video/video_frame_type.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+using ::testing::ElementsAre;
+using ::testing::IsEmpty;
+using ::testing::UnorderedElementsAre;
+
+constexpr VideoFrameType kVideoFrameKey = VideoFrameType::kVideoFrameKey;
+constexpr VideoFrameType kVideoFrameDelta = VideoFrameType::kVideoFrameDelta;
+
+constexpr CodecBufferUsage ReferenceAndUpdate(int id) {
+ return CodecBufferUsage(id, /*referenced=*/true, /*updated=*/true);
+}
+constexpr CodecBufferUsage Reference(int id) {
+ return CodecBufferUsage(id, /*referenced=*/true, /*updated=*/false);
+}
+constexpr CodecBufferUsage Update(int id) {
+ return CodecBufferUsage(id, /*referenced=*/false, /*updated=*/true);
+}
+
+TEST(FrameDependenciesCalculatorTest, SingleLayer) {
+ CodecBufferUsage pattern[] = {ReferenceAndUpdate(0)};
+ FrameDependenciesCalculator calculator;
+
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameKey, /*frame_id=*/1, pattern),
+ IsEmpty());
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/3, pattern),
+ ElementsAre(1));
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/6, pattern),
+ ElementsAre(3));
+}
+
+TEST(FrameDependenciesCalculatorTest, TwoTemporalLayers) {
+ // Shortened 4-frame pattern:
+ // T1: 2---4 6---8 ...
+ // / / / /
+ // T0: 1---3---5---7 ...
+ CodecBufferUsage pattern0[] = {ReferenceAndUpdate(0)};
+ CodecBufferUsage pattern1[] = {Reference(0), Update(1)};
+ CodecBufferUsage pattern2[] = {ReferenceAndUpdate(0)};
+ CodecBufferUsage pattern3[] = {Reference(0), Reference(1)};
+ FrameDependenciesCalculator calculator;
+
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameKey, /*frame_id=*/1, pattern0),
+ IsEmpty());
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/2, pattern1),
+ ElementsAre(1));
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/3, pattern2),
+ ElementsAre(1));
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/4, pattern3),
+ UnorderedElementsAre(2, 3));
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/5, pattern0),
+ ElementsAre(3));
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/6, pattern1),
+ ElementsAre(5));
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/7, pattern2),
+ ElementsAre(5));
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/8, pattern3),
+ UnorderedElementsAre(6, 7));
+}
+
+TEST(FrameDependenciesCalculatorTest, ThreeTemporalLayers4FramePattern) {
+ // T2: 2---4 6---8 ...
+ // / / / /
+ // T1: | 3 | 7 ...
+ // /_/ /_/
+ // T0: 1-------5----- ...
+ CodecBufferUsage pattern0[] = {ReferenceAndUpdate(0)};
+ CodecBufferUsage pattern1[] = {Reference(0), Update(2)};
+ CodecBufferUsage pattern2[] = {Reference(0), Update(1)};
+ CodecBufferUsage pattern3[] = {Reference(0), Reference(1), Reference(2)};
+ FrameDependenciesCalculator calculator;
+
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameKey, /*frame_id=*/1, pattern0),
+ IsEmpty());
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/2, pattern1),
+ ElementsAre(1));
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/3, pattern2),
+ ElementsAre(1));
+ // Note that frame#4 references buffer#0 that is updated by frame#1,
+ // yet there is no direct dependency from frame#4 to frame#1.
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/4, pattern3),
+ UnorderedElementsAre(2, 3));
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/5, pattern0),
+ ElementsAre(1));
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/6, pattern1),
+ ElementsAre(5));
+}
+
+TEST(FrameDependenciesCalculatorTest, SimulcastWith2Layers) {
+ // S1: 2---4---6- ...
+ //
+ // S0: 1---3---5- ...
+ CodecBufferUsage pattern0[] = {ReferenceAndUpdate(0)};
+ CodecBufferUsage pattern1[] = {ReferenceAndUpdate(1)};
+ FrameDependenciesCalculator calculator;
+
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameKey, /*frame_id=*/1, pattern0),
+ IsEmpty());
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameKey, /*frame_id=*/2, pattern1),
+ IsEmpty());
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/3, pattern0),
+ ElementsAre(1));
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/4, pattern1),
+ ElementsAre(2));
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/5, pattern0),
+ ElementsAre(3));
+ EXPECT_THAT(
+ calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/6, pattern1),
+ ElementsAre(4));
+}
+
+} // namespace
+} // namespace webrtc