ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 1 | /* |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 2 | * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 11 | #include "video/frame_encode_metadata_writer.h" |
| 12 | |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 13 | #include <cstddef> |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 14 | #include <vector> |
| 15 | |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 16 | #include "api/video/i420_buffer.h" |
| 17 | #include "api/video/video_frame.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 18 | #include "api/video/video_timing.h" |
Mirta Dvornicic | 28f0eb2 | 2019-05-28 16:30:16 +0200 | [diff] [blame] | 19 | #include "common_video/h264/h264_common.h" |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 20 | #include "common_video/test/utilities.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 21 | #include "modules/video_coding/include/video_coding_defines.h" |
Steve Anton | 10542f2 | 2019-01-11 09:11:00 -0800 | [diff] [blame] | 22 | #include "rtc_base/time_utils.h" |
Mirta Dvornicic | 28f0eb2 | 2019-05-28 16:30:16 +0200 | [diff] [blame] | 23 | #include "test/gmock.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 24 | #include "test/gtest.h" |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 25 | |
| 26 | namespace webrtc { |
| 27 | namespace test { |
| 28 | namespace { |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 29 | |
| 30 | const rtc::scoped_refptr<I420Buffer> kFrameBuffer = I420Buffer::Create(4, 4); |
| 31 | |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 32 | inline size_t FrameSize(const size_t& min_frame_size, |
| 33 | const size_t& max_frame_size, |
| 34 | const int& s, |
| 35 | const int& i) { |
| 36 | return min_frame_size + (s + 1) * i % (max_frame_size - min_frame_size); |
| 37 | } |
| 38 | |
| 39 | class FakeEncodedImageCallback : public EncodedImageCallback { |
| 40 | public: |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 41 | FakeEncodedImageCallback() : num_frames_dropped_(0) {} |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 42 | Result OnEncodedImage(const EncodedImage& encoded_image, |
| 43 | const CodecSpecificInfo* codec_specific_info, |
| 44 | const RTPFragmentationHeader* fragmentation) override { |
mflodman | 351424e | 2017-08-10 02:43:14 -0700 | [diff] [blame] | 45 | return Result(Result::OK); |
Mirko Bonadei | c4dd730 | 2019-02-25 09:12:02 +0100 | [diff] [blame] | 46 | } |
Ilya Nikolaevskiy | d79314f | 2017-10-23 10:45:37 +0200 | [diff] [blame] | 47 | void OnDroppedFrame(DropReason reason) override { ++num_frames_dropped_; } |
Ilya Nikolaevskiy | d79314f | 2017-10-23 10:45:37 +0200 | [diff] [blame] | 48 | size_t GetNumFramesDropped() { return num_frames_dropped_; } |
| 49 | |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 50 | private: |
Ilya Nikolaevskiy | d79314f | 2017-10-23 10:45:37 +0200 | [diff] [blame] | 51 | size_t num_frames_dropped_; |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 52 | }; |
| 53 | |
| 54 | enum class FrameType { |
| 55 | kNormal, |
| 56 | kTiming, |
| 57 | kDropped, |
| 58 | }; |
| 59 | |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 60 | bool IsTimingFrame(const EncodedImage& image) { |
| 61 | return image.timing_.flags != VideoSendTiming::kInvalid && |
| 62 | image.timing_.flags != VideoSendTiming::kNotTriggered; |
| 63 | } |
| 64 | |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 65 | // Emulates |num_frames| on |num_streams| frames with capture timestamps |
| 66 | // increased by 1 from 0. Size of each frame is between |
| 67 | // |min_frame_size| and |max_frame_size|, outliers are counted relatevely to |
| 68 | // |average_frame_sizes[]| for each stream. |
| 69 | std::vector<std::vector<FrameType>> GetTimingFrames( |
| 70 | const int64_t delay_ms, |
| 71 | const size_t min_frame_size, |
| 72 | const size_t max_frame_size, |
| 73 | std::vector<size_t> average_frame_sizes, |
| 74 | const int num_streams, |
| 75 | const int num_frames) { |
| 76 | FakeEncodedImageCallback sink; |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 77 | FrameEncodeMetadataWriter encode_timer(&sink); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 78 | VideoCodec codec_settings; |
| 79 | codec_settings.numberOfSimulcastStreams = num_streams; |
| 80 | codec_settings.timing_frame_thresholds = {delay_ms, |
| 81 | kDefaultOutlierFrameSizePercent}; |
| 82 | encode_timer.OnEncoderInit(codec_settings, false); |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 83 | const size_t kFramerate = 30; |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 84 | VideoBitrateAllocation bitrate_allocation; |
| 85 | for (int si = 0; si < num_streams; ++si) { |
| 86 | bitrate_allocation.SetBitrate(si, 0, |
| 87 | average_frame_sizes[si] * 8 * kFramerate); |
| 88 | } |
| 89 | encode_timer.OnSetRates(bitrate_allocation, kFramerate); |
| 90 | |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 91 | std::vector<std::vector<FrameType>> result(num_streams); |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 92 | int64_t current_timestamp = 0; |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 93 | for (int i = 0; i < num_frames; ++i) { |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 94 | current_timestamp += 1; |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 95 | VideoFrame frame = VideoFrame::Builder() |
| 96 | .set_timestamp_rtp(current_timestamp * 90) |
| 97 | .set_timestamp_ms(current_timestamp) |
| 98 | .set_video_frame_buffer(kFrameBuffer) |
| 99 | .build(); |
| 100 | encode_timer.OnEncodeStarted(frame); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 101 | for (int si = 0; si < num_streams; ++si) { |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 102 | // every (5+s)-th frame is dropped on s-th stream by design. |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 103 | bool dropped = i % (5 + si) == 0; |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 104 | |
| 105 | EncodedImage image; |
Niels Möller | 4d504c7 | 2019-06-18 15:56:56 +0200 | [diff] [blame] | 106 | image.SetEncodedData(EncodedImageBuffer::Create(max_frame_size)); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 107 | image.set_size(FrameSize(min_frame_size, max_frame_size, si, i)); |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 108 | image.capture_time_ms_ = current_timestamp; |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 109 | image.SetTimestamp(static_cast<uint32_t>(current_timestamp * 90)); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 110 | image.SetSpatialIndex(si); |
| 111 | |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 112 | if (dropped) { |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 113 | result[si].push_back(FrameType::kDropped); |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 114 | continue; |
| 115 | } |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 116 | |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 117 | encode_timer.FillTimingInfo(si, &image); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 118 | |
| 119 | if (IsTimingFrame(image)) { |
| 120 | result[si].push_back(FrameType::kTiming); |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 121 | } else { |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 122 | result[si].push_back(FrameType::kNormal); |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 123 | } |
| 124 | } |
| 125 | } |
| 126 | return result; |
| 127 | } |
| 128 | } // namespace |
| 129 | |
Mirta Dvornicic | 28f0eb2 | 2019-05-28 16:30:16 +0200 | [diff] [blame] | 130 | TEST(FrameEncodeMetadataWriterTest, MarksTimingFramesPeriodicallyTogether) { |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 131 | const int64_t kDelayMs = 29; |
| 132 | const size_t kMinFrameSize = 10; |
| 133 | const size_t kMaxFrameSize = 20; |
| 134 | const int kNumFrames = 1000; |
| 135 | const int kNumStreams = 3; |
| 136 | // No outliers as 1000 is larger than anything from range [10,20]. |
| 137 | const std::vector<size_t> kAverageSize = {1000, 1000, 1000}; |
| 138 | auto frames = GetTimingFrames(kDelayMs, kMinFrameSize, kMaxFrameSize, |
| 139 | kAverageSize, kNumStreams, kNumFrames); |
| 140 | // Timing frames should be tirggered every delayMs. |
| 141 | // As no outliers are expected, frames on all streams have to be |
| 142 | // marked together. |
| 143 | int last_timing_frame = -1; |
| 144 | for (int i = 0; i < kNumFrames; ++i) { |
| 145 | int num_normal = 0; |
| 146 | int num_timing = 0; |
| 147 | int num_dropped = 0; |
| 148 | for (int s = 0; s < kNumStreams; ++s) { |
| 149 | if (frames[s][i] == FrameType::kTiming) { |
| 150 | ++num_timing; |
| 151 | } else if (frames[s][i] == FrameType::kNormal) { |
| 152 | ++num_normal; |
| 153 | } else { |
| 154 | ++num_dropped; |
| 155 | } |
| 156 | } |
| 157 | // Can't have both normal and timing frames at the same timstamp. |
| 158 | EXPECT_TRUE(num_timing == 0 || num_normal == 0); |
| 159 | if (num_dropped < kNumStreams) { |
| 160 | if (last_timing_frame == -1 || i >= last_timing_frame + kDelayMs) { |
| 161 | // If didn't have timing frames for a period, current sent frame has to |
| 162 | // be one. No normal frames should be sent. |
| 163 | EXPECT_EQ(num_normal, 0); |
| 164 | } else { |
| 165 | // No unneeded timing frames should be sent. |
| 166 | EXPECT_EQ(num_timing, 0); |
| 167 | } |
| 168 | } |
| 169 | if (num_timing > 0) |
| 170 | last_timing_frame = i; |
| 171 | } |
| 172 | } |
| 173 | |
Mirta Dvornicic | 28f0eb2 | 2019-05-28 16:30:16 +0200 | [diff] [blame] | 174 | TEST(FrameEncodeMetadataWriterTest, MarksOutliers) { |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 175 | const int64_t kDelayMs = 29; |
| 176 | const size_t kMinFrameSize = 2495; |
| 177 | const size_t kMaxFrameSize = 2505; |
| 178 | const int kNumFrames = 1000; |
| 179 | const int kNumStreams = 3; |
| 180 | // Possible outliers as 1000 lies in range [995, 1005]. |
| 181 | const std::vector<size_t> kAverageSize = {998, 1000, 1004}; |
| 182 | auto frames = GetTimingFrames(kDelayMs, kMinFrameSize, kMaxFrameSize, |
| 183 | kAverageSize, kNumStreams, kNumFrames); |
| 184 | // All outliers should be marked. |
| 185 | for (int i = 0; i < kNumFrames; ++i) { |
| 186 | for (int s = 0; s < kNumStreams; ++s) { |
| 187 | if (FrameSize(kMinFrameSize, kMaxFrameSize, s, i) >= |
| 188 | kAverageSize[s] * kDefaultOutlierFrameSizePercent / 100) { |
| 189 | // Too big frame. May be dropped or timing, but not normal. |
| 190 | EXPECT_NE(frames[s][i], FrameType::kNormal); |
| 191 | } |
| 192 | } |
| 193 | } |
| 194 | } |
| 195 | |
Mirta Dvornicic | 28f0eb2 | 2019-05-28 16:30:16 +0200 | [diff] [blame] | 196 | TEST(FrameEncodeMetadataWriterTest, NoTimingFrameIfNoEncodeStartTime) { |
sprang | ba050a6 | 2017-08-18 02:51:12 -0700 | [diff] [blame] | 197 | int64_t timestamp = 1; |
Niels Möller | 663844d | 2019-02-14 16:15:54 +0100 | [diff] [blame] | 198 | constexpr size_t kFrameSize = 500; |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 199 | EncodedImage image; |
Niels Möller | 4d504c7 | 2019-06-18 15:56:56 +0200 | [diff] [blame] | 200 | image.SetEncodedData(EncodedImageBuffer::Create(kFrameSize)); |
sprang | ba050a6 | 2017-08-18 02:51:12 -0700 | [diff] [blame] | 201 | image.capture_time_ms_ = timestamp; |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 202 | image.SetTimestamp(static_cast<uint32_t>(timestamp * 90)); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 203 | |
sprang | ba050a6 | 2017-08-18 02:51:12 -0700 | [diff] [blame] | 204 | FakeEncodedImageCallback sink; |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 205 | FrameEncodeMetadataWriter encode_timer(&sink); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 206 | VideoCodec codec_settings; |
| 207 | // Make all frames timing frames. |
| 208 | codec_settings.timing_frame_thresholds.delay_ms = 1; |
| 209 | encode_timer.OnEncoderInit(codec_settings, false); |
| 210 | VideoBitrateAllocation bitrate_allocation; |
| 211 | bitrate_allocation.SetBitrate(0, 0, 500000); |
| 212 | encode_timer.OnSetRates(bitrate_allocation, 30); |
sprang | ba050a6 | 2017-08-18 02:51:12 -0700 | [diff] [blame] | 213 | |
| 214 | // Verify a single frame works with encode start time set. |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 215 | VideoFrame frame = VideoFrame::Builder() |
| 216 | .set_timestamp_ms(timestamp) |
| 217 | .set_timestamp_rtp(timestamp * 90) |
| 218 | .set_video_frame_buffer(kFrameBuffer) |
| 219 | .build(); |
| 220 | encode_timer.OnEncodeStarted(frame); |
| 221 | encode_timer.FillTimingInfo(0, &image); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 222 | EXPECT_TRUE(IsTimingFrame(image)); |
sprang | ba050a6 | 2017-08-18 02:51:12 -0700 | [diff] [blame] | 223 | |
| 224 | // New frame, now skip OnEncodeStarted. Should not result in timing frame. |
| 225 | image.capture_time_ms_ = ++timestamp; |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 226 | image.SetTimestamp(static_cast<uint32_t>(timestamp * 90)); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 227 | image.timing_ = EncodedImage::Timing(); |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 228 | encode_timer.FillTimingInfo(0, &image); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 229 | EXPECT_FALSE(IsTimingFrame(image)); |
sprang | ba050a6 | 2017-08-18 02:51:12 -0700 | [diff] [blame] | 230 | } |
| 231 | |
Mirta Dvornicic | 28f0eb2 | 2019-05-28 16:30:16 +0200 | [diff] [blame] | 232 | TEST(FrameEncodeMetadataWriterTest, |
| 233 | AdjustsCaptureTimeForInternalSourceEncoder) { |
Ilya Nikolaevskiy | 764aeb7 | 2018-04-03 10:01:52 +0200 | [diff] [blame] | 234 | const int64_t kEncodeStartDelayMs = 2; |
| 235 | const int64_t kEncodeFinishDelayMs = 10; |
Niels Möller | 663844d | 2019-02-14 16:15:54 +0100 | [diff] [blame] | 236 | constexpr size_t kFrameSize = 500; |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 237 | |
| 238 | int64_t timestamp = 1; |
| 239 | EncodedImage image; |
Niels Möller | 4d504c7 | 2019-06-18 15:56:56 +0200 | [diff] [blame] | 240 | image.SetEncodedData(EncodedImageBuffer::Create(kFrameSize)); |
Ilya Nikolaevskiy | 764aeb7 | 2018-04-03 10:01:52 +0200 | [diff] [blame] | 241 | image.capture_time_ms_ = timestamp; |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 242 | image.SetTimestamp(static_cast<uint32_t>(timestamp * 90)); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 243 | |
Ilya Nikolaevskiy | 764aeb7 | 2018-04-03 10:01:52 +0200 | [diff] [blame] | 244 | FakeEncodedImageCallback sink; |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 245 | FrameEncodeMetadataWriter encode_timer(&sink); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 246 | |
| 247 | VideoCodec codec_settings; |
| 248 | // Make all frames timing frames. |
| 249 | codec_settings.timing_frame_thresholds.delay_ms = 1; |
| 250 | encode_timer.OnEncoderInit(codec_settings, true); |
| 251 | |
| 252 | VideoBitrateAllocation bitrate_allocation; |
| 253 | bitrate_allocation.SetBitrate(0, 0, 500000); |
| 254 | encode_timer.OnSetRates(bitrate_allocation, 30); |
Ilya Nikolaevskiy | 764aeb7 | 2018-04-03 10:01:52 +0200 | [diff] [blame] | 255 | |
| 256 | // Verify a single frame without encode timestamps isn't a timing frame. |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 257 | encode_timer.FillTimingInfo(0, &image); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 258 | EXPECT_FALSE(IsTimingFrame(image)); |
Ilya Nikolaevskiy | 764aeb7 | 2018-04-03 10:01:52 +0200 | [diff] [blame] | 259 | |
| 260 | // New frame, but this time with encode timestamps set in timing_. |
| 261 | // This should be a timing frame. |
| 262 | image.capture_time_ms_ = ++timestamp; |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 263 | image.SetTimestamp(static_cast<uint32_t>(timestamp * 90)); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 264 | image.timing_ = EncodedImage::Timing(); |
Ilya Nikolaevskiy | 764aeb7 | 2018-04-03 10:01:52 +0200 | [diff] [blame] | 265 | image.timing_.encode_start_ms = timestamp + kEncodeStartDelayMs; |
| 266 | image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs; |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 267 | |
| 268 | encode_timer.FillTimingInfo(0, &image); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 269 | EXPECT_TRUE(IsTimingFrame(image)); |
| 270 | |
Ilya Nikolaevskiy | 764aeb7 | 2018-04-03 10:01:52 +0200 | [diff] [blame] | 271 | // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored |
| 272 | // capture timestamp should be kEncodeFinishDelayMs in the past. |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 273 | EXPECT_NEAR(image.capture_time_ms_, rtc::TimeMillis() - kEncodeFinishDelayMs, |
| 274 | 1); |
Ilya Nikolaevskiy | 764aeb7 | 2018-04-03 10:01:52 +0200 | [diff] [blame] | 275 | } |
| 276 | |
Mirta Dvornicic | 28f0eb2 | 2019-05-28 16:30:16 +0200 | [diff] [blame] | 277 | TEST(FrameEncodeMetadataWriterTest, NotifiesAboutDroppedFrames) { |
Ilya Nikolaevskiy | 76f2a85 | 2017-11-16 14:33:53 +0100 | [diff] [blame] | 278 | const int64_t kTimestampMs1 = 47721840; |
| 279 | const int64_t kTimestampMs2 = 47721850; |
| 280 | const int64_t kTimestampMs3 = 47721860; |
| 281 | const int64_t kTimestampMs4 = 47721870; |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 282 | |
Ilya Nikolaevskiy | d79314f | 2017-10-23 10:45:37 +0200 | [diff] [blame] | 283 | FakeEncodedImageCallback sink; |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 284 | FrameEncodeMetadataWriter encode_timer(&sink); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 285 | encode_timer.OnEncoderInit(VideoCodec(), false); |
Ilya Nikolaevskiy | e0da9ea | 2017-11-08 14:39:02 +0100 | [diff] [blame] | 286 | // Any non-zero bitrate needed to be set before the first frame. |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 287 | VideoBitrateAllocation bitrate_allocation; |
| 288 | bitrate_allocation.SetBitrate(0, 0, 500000); |
| 289 | encode_timer.OnSetRates(bitrate_allocation, 30); |
| 290 | |
| 291 | EncodedImage image; |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 292 | VideoFrame frame = VideoFrame::Builder() |
| 293 | .set_timestamp_rtp(kTimestampMs1 * 90) |
| 294 | .set_timestamp_ms(kTimestampMs1) |
| 295 | .set_video_frame_buffer(kFrameBuffer) |
| 296 | .build(); |
| 297 | |
Ilya Nikolaevskiy | 76f2a85 | 2017-11-16 14:33:53 +0100 | [diff] [blame] | 298 | image.capture_time_ms_ = kTimestampMs1; |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 299 | image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90)); |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 300 | frame.set_timestamp(image.capture_time_ms_ * 90); |
| 301 | frame.set_timestamp_us(image.capture_time_ms_ * 1000); |
| 302 | encode_timer.OnEncodeStarted(frame); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 303 | |
Ilya Nikolaevskiy | d79314f | 2017-10-23 10:45:37 +0200 | [diff] [blame] | 304 | EXPECT_EQ(0u, sink.GetNumFramesDropped()); |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 305 | encode_timer.FillTimingInfo(0, &image); |
Ilya Nikolaevskiy | 76f2a85 | 2017-11-16 14:33:53 +0100 | [diff] [blame] | 306 | |
| 307 | image.capture_time_ms_ = kTimestampMs2; |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 308 | image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90)); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 309 | image.timing_ = EncodedImage::Timing(); |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 310 | frame.set_timestamp(image.capture_time_ms_ * 90); |
| 311 | frame.set_timestamp_us(image.capture_time_ms_ * 1000); |
| 312 | encode_timer.OnEncodeStarted(frame); |
Ilya Nikolaevskiy | d79314f | 2017-10-23 10:45:37 +0200 | [diff] [blame] | 313 | // No OnEncodedImageCall for timestamp2. Yet, at this moment it's not known |
| 314 | // that frame with timestamp2 was dropped. |
| 315 | EXPECT_EQ(0u, sink.GetNumFramesDropped()); |
Ilya Nikolaevskiy | 76f2a85 | 2017-11-16 14:33:53 +0100 | [diff] [blame] | 316 | |
| 317 | image.capture_time_ms_ = kTimestampMs3; |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 318 | image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90)); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 319 | image.timing_ = EncodedImage::Timing(); |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 320 | frame.set_timestamp(image.capture_time_ms_ * 90); |
| 321 | frame.set_timestamp_us(image.capture_time_ms_ * 1000); |
| 322 | encode_timer.OnEncodeStarted(frame); |
| 323 | encode_timer.FillTimingInfo(0, &image); |
Ilya Nikolaevskiy | d79314f | 2017-10-23 10:45:37 +0200 | [diff] [blame] | 324 | EXPECT_EQ(1u, sink.GetNumFramesDropped()); |
Ilya Nikolaevskiy | 76f2a85 | 2017-11-16 14:33:53 +0100 | [diff] [blame] | 325 | |
| 326 | image.capture_time_ms_ = kTimestampMs4; |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 327 | image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90)); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 328 | image.timing_ = EncodedImage::Timing(); |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 329 | frame.set_timestamp(image.capture_time_ms_ * 90); |
| 330 | frame.set_timestamp_us(image.capture_time_ms_ * 1000); |
| 331 | encode_timer.OnEncodeStarted(frame); |
| 332 | encode_timer.FillTimingInfo(0, &image); |
Ilya Nikolaevskiy | d79314f | 2017-10-23 10:45:37 +0200 | [diff] [blame] | 333 | EXPECT_EQ(1u, sink.GetNumFramesDropped()); |
| 334 | } |
| 335 | |
Mirta Dvornicic | 28f0eb2 | 2019-05-28 16:30:16 +0200 | [diff] [blame] | 336 | TEST(FrameEncodeMetadataWriterTest, RestoresCaptureTimestamps) { |
Ilya Nikolaevskiy | 76f2a85 | 2017-11-16 14:33:53 +0100 | [diff] [blame] | 337 | EncodedImage image; |
Ilya Nikolaevskiy | 76f2a85 | 2017-11-16 14:33:53 +0100 | [diff] [blame] | 338 | const int64_t kTimestampMs = 123456; |
Ilya Nikolaevskiy | 76f2a85 | 2017-11-16 14:33:53 +0100 | [diff] [blame] | 339 | FakeEncodedImageCallback sink; |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 340 | |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 341 | FrameEncodeMetadataWriter encode_timer(&sink); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 342 | encode_timer.OnEncoderInit(VideoCodec(), false); |
Ilya Nikolaevskiy | 76f2a85 | 2017-11-16 14:33:53 +0100 | [diff] [blame] | 343 | // Any non-zero bitrate needed to be set before the first frame. |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 344 | VideoBitrateAllocation bitrate_allocation; |
| 345 | bitrate_allocation.SetBitrate(0, 0, 500000); |
| 346 | encode_timer.OnSetRates(bitrate_allocation, 30); |
| 347 | |
| 348 | image.capture_time_ms_ = kTimestampMs; // Correct timestamp. |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 349 | image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90)); |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 350 | VideoFrame frame = VideoFrame::Builder() |
| 351 | .set_timestamp_ms(image.capture_time_ms_) |
| 352 | .set_timestamp_rtp(image.capture_time_ms_ * 90) |
| 353 | .set_video_frame_buffer(kFrameBuffer) |
| 354 | .build(); |
| 355 | encode_timer.OnEncodeStarted(frame); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 356 | image.capture_time_ms_ = 0; // Incorrect timestamp. |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 357 | encode_timer.FillTimingInfo(0, &image); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 358 | EXPECT_EQ(kTimestampMs, image.capture_time_ms_); |
Ilya Nikolaevskiy | 76f2a85 | 2017-11-16 14:33:53 +0100 | [diff] [blame] | 359 | } |
| 360 | |
Mirta Dvornicic | 28f0eb2 | 2019-05-28 16:30:16 +0200 | [diff] [blame] | 361 | TEST(FrameEncodeMetadataWriterTest, CopiesRotation) { |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 362 | EncodedImage image; |
| 363 | const int64_t kTimestampMs = 123456; |
| 364 | FakeEncodedImageCallback sink; |
| 365 | |
| 366 | FrameEncodeMetadataWriter encode_timer(&sink); |
| 367 | encode_timer.OnEncoderInit(VideoCodec(), false); |
| 368 | // Any non-zero bitrate needed to be set before the first frame. |
| 369 | VideoBitrateAllocation bitrate_allocation; |
| 370 | bitrate_allocation.SetBitrate(0, 0, 500000); |
| 371 | encode_timer.OnSetRates(bitrate_allocation, 30); |
| 372 | |
| 373 | image.SetTimestamp(static_cast<uint32_t>(kTimestampMs * 90)); |
| 374 | VideoFrame frame = VideoFrame::Builder() |
| 375 | .set_timestamp_ms(kTimestampMs) |
| 376 | .set_timestamp_rtp(kTimestampMs * 90) |
| 377 | .set_rotation(kVideoRotation_180) |
| 378 | .set_video_frame_buffer(kFrameBuffer) |
| 379 | .build(); |
| 380 | encode_timer.OnEncodeStarted(frame); |
| 381 | encode_timer.FillTimingInfo(0, &image); |
| 382 | EXPECT_EQ(kVideoRotation_180, image.rotation_); |
| 383 | } |
| 384 | |
Mirta Dvornicic | 28f0eb2 | 2019-05-28 16:30:16 +0200 | [diff] [blame] | 385 | TEST(FrameEncodeMetadataWriterTest, SetsContentType) { |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 386 | EncodedImage image; |
| 387 | const int64_t kTimestampMs = 123456; |
| 388 | FakeEncodedImageCallback sink; |
| 389 | |
| 390 | FrameEncodeMetadataWriter encode_timer(&sink); |
| 391 | VideoCodec codec; |
| 392 | codec.mode = VideoCodecMode::kScreensharing; |
| 393 | encode_timer.OnEncoderInit(codec, false); |
| 394 | // Any non-zero bitrate needed to be set before the first frame. |
| 395 | VideoBitrateAllocation bitrate_allocation; |
| 396 | bitrate_allocation.SetBitrate(0, 0, 500000); |
| 397 | encode_timer.OnSetRates(bitrate_allocation, 30); |
| 398 | |
| 399 | image.SetTimestamp(static_cast<uint32_t>(kTimestampMs * 90)); |
| 400 | VideoFrame frame = VideoFrame::Builder() |
| 401 | .set_timestamp_ms(kTimestampMs) |
| 402 | .set_timestamp_rtp(kTimestampMs * 90) |
| 403 | .set_rotation(kVideoRotation_180) |
| 404 | .set_video_frame_buffer(kFrameBuffer) |
| 405 | .build(); |
| 406 | encode_timer.OnEncodeStarted(frame); |
| 407 | encode_timer.FillTimingInfo(0, &image); |
| 408 | EXPECT_EQ(VideoContentType::SCREENSHARE, image.content_type_); |
| 409 | } |
| 410 | |
Mirta Dvornicic | 28f0eb2 | 2019-05-28 16:30:16 +0200 | [diff] [blame] | 411 | TEST(FrameEncodeMetadataWriterTest, CopiesColorSpace) { |
Ilya Nikolaevskiy | 2ebf523 | 2019-05-13 16:13:36 +0200 | [diff] [blame] | 412 | EncodedImage image; |
| 413 | const int64_t kTimestampMs = 123456; |
| 414 | FakeEncodedImageCallback sink; |
| 415 | |
| 416 | FrameEncodeMetadataWriter encode_timer(&sink); |
| 417 | encode_timer.OnEncoderInit(VideoCodec(), false); |
| 418 | // Any non-zero bitrate needed to be set before the first frame. |
| 419 | VideoBitrateAllocation bitrate_allocation; |
| 420 | bitrate_allocation.SetBitrate(0, 0, 500000); |
| 421 | encode_timer.OnSetRates(bitrate_allocation, 30); |
| 422 | |
| 423 | webrtc::ColorSpace color_space = |
| 424 | CreateTestColorSpace(/*with_hdr_metadata=*/true); |
| 425 | image.SetTimestamp(static_cast<uint32_t>(kTimestampMs * 90)); |
| 426 | VideoFrame frame = VideoFrame::Builder() |
| 427 | .set_timestamp_ms(kTimestampMs) |
| 428 | .set_timestamp_rtp(kTimestampMs * 90) |
| 429 | .set_color_space(color_space) |
| 430 | .set_video_frame_buffer(kFrameBuffer) |
| 431 | .build(); |
| 432 | encode_timer.OnEncodeStarted(frame); |
| 433 | encode_timer.FillTimingInfo(0, &image); |
| 434 | ASSERT_NE(image.ColorSpace(), nullptr); |
| 435 | EXPECT_EQ(color_space, *image.ColorSpace()); |
| 436 | } |
| 437 | |
Chen Xing | f00bf42 | 2019-06-20 10:05:55 +0200 | [diff] [blame] | 438 | TEST(FrameEncodeMetadataWriterTest, CopiesPacketInfos) { |
| 439 | EncodedImage image; |
| 440 | const int64_t kTimestampMs = 123456; |
| 441 | FakeEncodedImageCallback sink; |
| 442 | |
| 443 | FrameEncodeMetadataWriter encode_timer(&sink); |
| 444 | encode_timer.OnEncoderInit(VideoCodec(), false); |
| 445 | // Any non-zero bitrate needed to be set before the first frame. |
| 446 | VideoBitrateAllocation bitrate_allocation; |
| 447 | bitrate_allocation.SetBitrate(0, 0, 500000); |
| 448 | encode_timer.OnSetRates(bitrate_allocation, 30); |
| 449 | |
| 450 | RtpPacketInfos packet_infos = CreatePacketInfos(3); |
| 451 | image.SetTimestamp(static_cast<uint32_t>(kTimestampMs * 90)); |
| 452 | VideoFrame frame = VideoFrame::Builder() |
| 453 | .set_timestamp_ms(kTimestampMs) |
| 454 | .set_timestamp_rtp(kTimestampMs * 90) |
| 455 | .set_packet_infos(packet_infos) |
| 456 | .set_video_frame_buffer(kFrameBuffer) |
| 457 | .build(); |
| 458 | encode_timer.OnEncodeStarted(frame); |
| 459 | encode_timer.FillTimingInfo(0, &image); |
| 460 | EXPECT_EQ(image.PacketInfos().size(), 3U); |
| 461 | } |
| 462 | |
Mirta Dvornicic | 28f0eb2 | 2019-05-28 16:30:16 +0200 | [diff] [blame] | 463 | TEST(FrameEncodeMetadataWriterTest, DoesNotRewriteBitstreamWithoutCodecInfo) { |
| 464 | uint8_t buffer[] = {1, 2, 3}; |
| 465 | EncodedImage image(buffer, sizeof(buffer), sizeof(buffer)); |
| 466 | const RTPFragmentationHeader fragmentation; |
| 467 | |
| 468 | FakeEncodedImageCallback sink; |
| 469 | FrameEncodeMetadataWriter encode_metadata_writer(&sink); |
| 470 | EXPECT_EQ( |
| 471 | encode_metadata_writer.UpdateBitstream(nullptr, &fragmentation, &image), |
| 472 | nullptr); |
| 473 | EXPECT_EQ(image.data(), buffer); |
| 474 | EXPECT_EQ(image.size(), sizeof(buffer)); |
| 475 | } |
| 476 | |
| 477 | TEST(FrameEncodeMetadataWriterTest, DoesNotRewriteVp8Bitstream) { |
| 478 | uint8_t buffer[] = {1, 2, 3}; |
| 479 | EncodedImage image(buffer, sizeof(buffer), sizeof(buffer)); |
| 480 | CodecSpecificInfo codec_specific_info; |
| 481 | codec_specific_info.codecType = kVideoCodecVP8; |
| 482 | const RTPFragmentationHeader fragmentation; |
| 483 | |
| 484 | FakeEncodedImageCallback sink; |
| 485 | FrameEncodeMetadataWriter encode_metadata_writer(&sink); |
| 486 | EXPECT_EQ(encode_metadata_writer.UpdateBitstream(&codec_specific_info, |
| 487 | &fragmentation, &image), |
| 488 | nullptr); |
| 489 | EXPECT_EQ(image.data(), buffer); |
| 490 | EXPECT_EQ(image.size(), sizeof(buffer)); |
| 491 | } |
| 492 | |
| 493 | TEST(FrameEncodeMetadataWriterTest, |
| 494 | DoesNotRewriteH264BitstreamWithoutFragmentation) { |
| 495 | uint8_t buffer[] = {1, 2, 3}; |
| 496 | EncodedImage image(buffer, sizeof(buffer), sizeof(buffer)); |
| 497 | CodecSpecificInfo codec_specific_info; |
| 498 | codec_specific_info.codecType = kVideoCodecH264; |
| 499 | |
| 500 | FakeEncodedImageCallback sink; |
| 501 | FrameEncodeMetadataWriter encode_metadata_writer(&sink); |
| 502 | EXPECT_EQ(encode_metadata_writer.UpdateBitstream(&codec_specific_info, |
| 503 | nullptr, &image), |
| 504 | nullptr); |
| 505 | EXPECT_EQ(image.data(), buffer); |
| 506 | EXPECT_EQ(image.size(), sizeof(buffer)); |
| 507 | } |
| 508 | |
| 509 | TEST(FrameEncodeMetadataWriterTest, RewritesH264BitstreamWithNonOptimalSps) { |
| 510 | uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps, |
| 511 | 0x00, 0x00, 0x03, 0x03, 0xF4, |
| 512 | 0x05, 0x03, 0xC7, 0xC0}; |
| 513 | const uint8_t kRewrittenSps[] = {0, 0, 0, 1, H264::NaluType::kSps, |
| 514 | 0x00, 0x00, 0x03, 0x03, 0xF4, |
| 515 | 0x05, 0x03, 0xC7, 0xE0, 0x1B, |
| 516 | 0x41, 0x10, 0x8D, 0x00}; |
| 517 | |
| 518 | EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps)); |
| 519 | image._frameType = VideoFrameType::kVideoFrameKey; |
| 520 | |
| 521 | CodecSpecificInfo codec_specific_info; |
| 522 | codec_specific_info.codecType = kVideoCodecH264; |
| 523 | |
| 524 | RTPFragmentationHeader fragmentation; |
| 525 | fragmentation.VerifyAndAllocateFragmentationHeader(1); |
| 526 | fragmentation.fragmentationOffset[0] = 4; |
| 527 | fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4; |
| 528 | |
| 529 | FakeEncodedImageCallback sink; |
| 530 | FrameEncodeMetadataWriter encode_metadata_writer(&sink); |
| 531 | std::unique_ptr<RTPFragmentationHeader> modified_fragmentation = |
| 532 | encode_metadata_writer.UpdateBitstream(&codec_specific_info, |
| 533 | &fragmentation, &image); |
| 534 | |
| 535 | ASSERT_NE(modified_fragmentation, nullptr); |
| 536 | EXPECT_THAT(std::vector<uint8_t>(image.data(), image.data() + image.size()), |
| 537 | testing::ElementsAreArray(kRewrittenSps)); |
| 538 | ASSERT_THAT(modified_fragmentation->fragmentationVectorSize, 1U); |
| 539 | EXPECT_EQ(modified_fragmentation->fragmentationOffset[0], 4U); |
| 540 | EXPECT_EQ(modified_fragmentation->fragmentationLength[0], |
| 541 | sizeof(kRewrittenSps) - 4); |
| 542 | } |
| 543 | |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 544 | } // namespace test |
| 545 | } // namespace webrtc |