kjellander@webrtc.org | 35a1756 | 2011-10-06 06:44:54 +0000 | [diff] [blame] | 1 | /* |
pwestin@webrtc.org | ce33035 | 2012-04-12 06:59:14 +0000 | [diff] [blame] | 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
kjellander@webrtc.org | 35a1756 | 2011-10-06 06:44:54 +0000 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
kjellander@webrtc.org | 5b97b12 | 2011-12-08 07:42:18 +0000 | [diff] [blame] | 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #include "modules/video_coding/codecs/test/videoprocessor.h" |
kjellander@webrtc.org | 35a1756 | 2011-10-06 06:44:54 +0000 | [diff] [blame] | 12 | |
ssilkin | 612f858 | 2017-09-28 09:23:17 -0700 | [diff] [blame] | 13 | #include <algorithm> |
kjellander@webrtc.org | 35a1756 | 2011-10-06 06:44:54 +0000 | [diff] [blame] | 14 | #include <limits> |
Erik Språng | 08127a9 | 2016-11-16 16:41:30 +0100 | [diff] [blame] | 15 | #include <utility> |
kjellander@webrtc.org | 35a1756 | 2011-10-06 06:44:54 +0000 | [diff] [blame] | 16 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 17 | #include "api/video/i420_buffer.h" |
Mirko Bonadei | 7120742 | 2017-09-15 13:58:09 +0200 | [diff] [blame] | 18 | #include "common_types.h" // NOLINT(build/include) |
ssilkin | 612f858 | 2017-09-28 09:23:17 -0700 | [diff] [blame] | 19 | #include "common_video/h264/h264_common.h" |
Rasmus Brandt | 5f7a891 | 2018-02-28 17:17:15 +0100 | [diff] [blame] | 20 | #include "common_video/libyuv/include/webrtc_libyuv.h" |
Sergey Silkin | 3be2a55 | 2018-01-17 15:11:44 +0100 | [diff] [blame] | 21 | #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" |
Mirko Bonadei | 6f440ed | 2018-06-21 13:41:01 +0000 | [diff] [blame^] | 22 | #include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 23 | #include "modules/video_coding/include/video_codec_initializer.h" |
Rasmus Brandt | d00c895 | 2018-03-14 12:29:57 +0100 | [diff] [blame] | 24 | #include "modules/video_coding/include/video_error_codes.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 25 | #include "modules/video_coding/utility/default_video_bitrate_allocator.h" |
| 26 | #include "rtc_base/checks.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 27 | #include "rtc_base/timeutils.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 28 | #include "test/gtest.h" |
Sergey Silkin | 8d3758e | 2018-03-14 11:28:15 +0100 | [diff] [blame] | 29 | #include "third_party/libyuv/include/libyuv/compare.h" |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 30 | #include "third_party/libyuv/include/libyuv/scale.h" |
kjellander@webrtc.org | 35a1756 | 2011-10-06 06:44:54 +0000 | [diff] [blame] | 31 | |
| 32 | namespace webrtc { |
| 33 | namespace test { |
| 34 | |
Kári Tristan Helgason | 169005d | 2018-05-22 13:34:14 +0200 | [diff] [blame] | 35 | using FrameStatistics = VideoCodecTestStats::FrameStatistics; |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 36 | |
Kári Tristan Helgason | 169005d | 2018-05-22 13:34:14 +0200 | [diff] [blame] | 37 | namespace { |
Åsa Persson | 91af24a | 2018-01-24 17:20:18 +0100 | [diff] [blame] | 38 | const int kMsToRtpTimestamp = kVideoPayloadTypeFrequency / 1000; |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 39 | const int kMaxBufferedInputFrames = 10; |
brandtr | 17b958c | 2017-03-07 01:41:43 -0800 | [diff] [blame] | 40 | |
Sergey Silkin | 3be2a55 | 2018-01-17 15:11:44 +0100 | [diff] [blame] | 41 | size_t GetMaxNaluSizeBytes(const EncodedImage& encoded_frame, |
Kári Tristan Helgason | 169005d | 2018-05-22 13:34:14 +0200 | [diff] [blame] | 42 | const VideoCodecTestFixture::Config& config) { |
ssilkin | 612f858 | 2017-09-28 09:23:17 -0700 | [diff] [blame] | 43 | if (config.codec_settings.codecType != kVideoCodecH264) |
Sergey Silkin | 3be2a55 | 2018-01-17 15:11:44 +0100 | [diff] [blame] | 44 | return 0; |
ssilkin | 612f858 | 2017-09-28 09:23:17 -0700 | [diff] [blame] | 45 | |
| 46 | std::vector<webrtc::H264::NaluIndex> nalu_indices = |
| 47 | webrtc::H264::FindNaluIndices(encoded_frame._buffer, |
| 48 | encoded_frame._length); |
| 49 | |
| 50 | RTC_CHECK(!nalu_indices.empty()); |
| 51 | |
Sergey Silkin | 3be2a55 | 2018-01-17 15:11:44 +0100 | [diff] [blame] | 52 | size_t max_size = 0; |
ssilkin | 612f858 | 2017-09-28 09:23:17 -0700 | [diff] [blame] | 53 | for (const webrtc::H264::NaluIndex& index : nalu_indices) |
Sergey Silkin | 3be2a55 | 2018-01-17 15:11:44 +0100 | [diff] [blame] | 54 | max_size = std::max(max_size, index.payload_size); |
ssilkin | 612f858 | 2017-09-28 09:23:17 -0700 | [diff] [blame] | 55 | |
Sergey Silkin | 3be2a55 | 2018-01-17 15:11:44 +0100 | [diff] [blame] | 56 | return max_size; |
ssilkin | 612f858 | 2017-09-28 09:23:17 -0700 | [diff] [blame] | 57 | } |
| 58 | |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 59 | void GetLayerIndices(const CodecSpecificInfo& codec_specific, |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 60 | size_t* spatial_idx, |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 61 | size_t* temporal_idx) { |
| 62 | if (codec_specific.codecType == kVideoCodecVP8) { |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 63 | *spatial_idx = codec_specific.codecSpecific.VP8.simulcastIdx; |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 64 | *temporal_idx = codec_specific.codecSpecific.VP8.temporalIdx; |
| 65 | } else if (codec_specific.codecType == kVideoCodecVP9) { |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 66 | *spatial_idx = codec_specific.codecSpecific.VP9.spatial_idx; |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 67 | *temporal_idx = codec_specific.codecSpecific.VP9.temporal_idx; |
| 68 | } |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 69 | if (*spatial_idx == kNoSpatialIdx) { |
| 70 | *spatial_idx = 0; |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 71 | } |
| 72 | if (*temporal_idx == kNoTemporalIdx) { |
| 73 | *temporal_idx = 0; |
| 74 | } |
| 75 | } |
| 76 | |
asapersson | ae9ba04 | 2017-03-07 00:25:38 -0800 | [diff] [blame] | 77 | int GetElapsedTimeMicroseconds(int64_t start_ns, int64_t stop_ns) { |
| 78 | int64_t diff_us = (stop_ns - start_ns) / rtc::kNumNanosecsPerMicrosec; |
| 79 | RTC_DCHECK_GE(diff_us, std::numeric_limits<int>::min()); |
| 80 | RTC_DCHECK_LE(diff_us, std::numeric_limits<int>::max()); |
| 81 | return static_cast<int>(diff_us); |
| 82 | } |
| 83 | |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 84 | void ExtractI420BufferWithSize(const VideoFrame& image, |
| 85 | int width, |
| 86 | int height, |
| 87 | rtc::Buffer* buffer) { |
Åsa Persson | f0c4467 | 2017-10-24 16:03:39 +0200 | [diff] [blame] | 88 | if (image.width() != width || image.height() != height) { |
| 89 | EXPECT_DOUBLE_EQ(static_cast<double>(width) / height, |
| 90 | static_cast<double>(image.width()) / image.height()); |
| 91 | // Same aspect ratio, no cropping needed. |
| 92 | rtc::scoped_refptr<I420Buffer> scaled(I420Buffer::Create(width, height)); |
| 93 | scaled->ScaleFrom(*image.video_frame_buffer()->ToI420()); |
| 94 | |
| 95 | size_t length = |
| 96 | CalcBufferSize(VideoType::kI420, scaled->width(), scaled->height()); |
| 97 | buffer->SetSize(length); |
| 98 | RTC_CHECK_NE(ExtractBuffer(scaled, length, buffer->data()), -1); |
| 99 | return; |
| 100 | } |
| 101 | |
| 102 | // No resize. |
| 103 | size_t length = |
| 104 | CalcBufferSize(VideoType::kI420, image.width(), image.height()); |
| 105 | buffer->SetSize(length); |
| 106 | RTC_CHECK_NE(ExtractBuffer(image, length, buffer->data()), -1); |
| 107 | } |
| 108 | |
Sergey Silkin | 8d3758e | 2018-03-14 11:28:15 +0100 | [diff] [blame] | 109 | void CalculateFrameQuality(const I420BufferInterface& ref_buffer, |
| 110 | const I420BufferInterface& dec_buffer, |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 111 | FrameStatistics* frame_stat) { |
Sergey Silkin | 8d3758e | 2018-03-14 11:28:15 +0100 | [diff] [blame] | 112 | if (ref_buffer.width() != dec_buffer.width() || |
| 113 | ref_buffer.height() != dec_buffer.height()) { |
| 114 | RTC_CHECK_GE(ref_buffer.width(), dec_buffer.width()); |
| 115 | RTC_CHECK_GE(ref_buffer.height(), dec_buffer.height()); |
| 116 | // Downscale reference frame. |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 117 | rtc::scoped_refptr<I420Buffer> scaled_buffer = |
Sergey Silkin | 8d3758e | 2018-03-14 11:28:15 +0100 | [diff] [blame] | 118 | I420Buffer::Create(dec_buffer.width(), dec_buffer.height()); |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 119 | I420Scale(ref_buffer.DataY(), ref_buffer.StrideY(), ref_buffer.DataU(), |
| 120 | ref_buffer.StrideU(), ref_buffer.DataV(), ref_buffer.StrideV(), |
| 121 | ref_buffer.width(), ref_buffer.height(), |
| 122 | scaled_buffer->MutableDataY(), scaled_buffer->StrideY(), |
| 123 | scaled_buffer->MutableDataU(), scaled_buffer->StrideU(), |
| 124 | scaled_buffer->MutableDataV(), scaled_buffer->StrideV(), |
| 125 | scaled_buffer->width(), scaled_buffer->height(), |
| 126 | libyuv::kFilterBox); |
Sergey Silkin | 8d3758e | 2018-03-14 11:28:15 +0100 | [diff] [blame] | 127 | |
| 128 | CalculateFrameQuality(*scaled_buffer, dec_buffer, frame_stat); |
| 129 | } else { |
| 130 | const uint64_t sse_y = libyuv::ComputeSumSquareErrorPlane( |
| 131 | dec_buffer.DataY(), dec_buffer.StrideY(), ref_buffer.DataY(), |
| 132 | ref_buffer.StrideY(), dec_buffer.width(), dec_buffer.height()); |
| 133 | |
| 134 | const uint64_t sse_u = libyuv::ComputeSumSquareErrorPlane( |
| 135 | dec_buffer.DataU(), dec_buffer.StrideU(), ref_buffer.DataU(), |
| 136 | ref_buffer.StrideU(), dec_buffer.width() / 2, dec_buffer.height() / 2); |
| 137 | |
| 138 | const uint64_t sse_v = libyuv::ComputeSumSquareErrorPlane( |
| 139 | dec_buffer.DataV(), dec_buffer.StrideV(), ref_buffer.DataV(), |
| 140 | ref_buffer.StrideV(), dec_buffer.width() / 2, dec_buffer.height() / 2); |
| 141 | |
| 142 | const size_t num_y_samples = dec_buffer.width() * dec_buffer.height(); |
| 143 | const size_t num_u_samples = |
| 144 | dec_buffer.width() / 2 * dec_buffer.height() / 2; |
| 145 | |
| 146 | frame_stat->psnr_y = libyuv::SumSquareErrorToPsnr(sse_y, num_y_samples); |
| 147 | frame_stat->psnr_u = libyuv::SumSquareErrorToPsnr(sse_u, num_u_samples); |
| 148 | frame_stat->psnr_v = libyuv::SumSquareErrorToPsnr(sse_v, num_u_samples); |
| 149 | frame_stat->psnr = libyuv::SumSquareErrorToPsnr( |
| 150 | sse_y + sse_u + sse_v, num_y_samples + 2 * num_u_samples); |
| 151 | frame_stat->ssim = I420SSIM(ref_buffer, dec_buffer); |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 152 | } |
| 153 | } |
| 154 | |
Kári Tristan Helgason | 169005d | 2018-05-22 13:34:14 +0200 | [diff] [blame] | 155 | std::vector<FrameType> FrameTypeForFrame( |
| 156 | const VideoCodecTestFixture::Config& config, |
| 157 | size_t frame_idx) { |
| 158 | if (config.keyframe_interval > 0 && |
| 159 | (frame_idx % config.keyframe_interval == 0)) { |
| 160 | return {kVideoFrameKey}; |
| 161 | } |
| 162 | return {kVideoFrameDelta}; |
| 163 | } |
| 164 | |
brandtr | b78bc75 | 2017-02-22 01:26:59 -0800 | [diff] [blame] | 165 | } // namespace |
| 166 | |
brandtr | c409552 | 2017-08-07 08:12:33 -0700 | [diff] [blame] | 167 | VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder, |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 168 | VideoDecoderList* decoders, |
| 169 | FrameReader* input_frame_reader, |
Kári Tristan Helgason | 169005d | 2018-05-22 13:34:14 +0200 | [diff] [blame] | 170 | const VideoCodecTestFixture::Config& config, |
| 171 | VideoCodecTestStats* stats, |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 172 | IvfFileWriterList* encoded_frame_writers, |
| 173 | FrameWriterList* decoded_frame_writers) |
Åsa Persson | f0c4467 | 2017-10-24 16:03:39 +0200 | [diff] [blame] | 174 | : config_(config), |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 175 | num_simulcast_or_spatial_layers_( |
| 176 | std::max(config_.NumberOfSimulcastStreams(), |
| 177 | config_.NumberOfSpatialLayers())), |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 178 | stats_(stats), |
brandtr | 07734a5 | 2017-08-08 08:35:53 -0700 | [diff] [blame] | 179 | encoder_(encoder), |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 180 | decoders_(decoders), |
Erik Språng | 82fad3d | 2018-03-21 09:57:23 +0100 | [diff] [blame] | 181 | bitrate_allocator_(VideoCodecInitializer::CreateBitrateAllocator( |
| 182 | config_.codec_settings)), |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 183 | framerate_fps_(0), |
brandtr | bdd555c | 2017-08-21 01:34:04 -0700 | [diff] [blame] | 184 | encode_callback_(this), |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 185 | input_frame_reader_(input_frame_reader), |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 186 | merged_encoded_frames_(num_simulcast_or_spatial_layers_), |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 187 | encoded_frame_writers_(encoded_frame_writers), |
| 188 | decoded_frame_writers_(decoded_frame_writers), |
Sergey Silkin | 3be2a55 | 2018-01-17 15:11:44 +0100 | [diff] [blame] | 189 | last_inputed_frame_num_(0), |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 190 | last_inputed_timestamp_(0), |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 191 | first_encoded_frame_(num_simulcast_or_spatial_layers_, true), |
| 192 | last_encoded_frame_num_(num_simulcast_or_spatial_layers_), |
| 193 | first_decoded_frame_(num_simulcast_or_spatial_layers_, true), |
Sergey Silkin | c89eed9 | 2018-04-01 23:57:51 +0200 | [diff] [blame] | 194 | last_decoded_frame_num_(num_simulcast_or_spatial_layers_), |
Sergey Silkin | 5613879 | 2018-05-02 10:50:55 +0200 | [diff] [blame] | 195 | decoded_frame_buffer_(num_simulcast_or_spatial_layers_), |
Sergey Silkin | c89eed9 | 2018-04-01 23:57:51 +0200 | [diff] [blame] | 196 | post_encode_time_ns_(0) { |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 197 | // Sanity checks. |
Rasmus Brandt | 4b381af | 2018-02-07 13:56:16 +0100 | [diff] [blame] | 198 | RTC_CHECK(rtc::TaskQueue::Current()) |
| 199 | << "VideoProcessor must be run on a task queue."; |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 200 | RTC_CHECK(encoder); |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 201 | RTC_CHECK(decoders); |
| 202 | RTC_CHECK_EQ(decoders->size(), num_simulcast_or_spatial_layers_); |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 203 | RTC_CHECK(input_frame_reader); |
| 204 | RTC_CHECK(stats); |
| 205 | RTC_CHECK(!encoded_frame_writers || |
| 206 | encoded_frame_writers->size() == num_simulcast_or_spatial_layers_); |
| 207 | RTC_CHECK(!decoded_frame_writers || |
| 208 | decoded_frame_writers->size() == num_simulcast_or_spatial_layers_); |
brandtr | 17b958c | 2017-03-07 01:41:43 -0800 | [diff] [blame] | 209 | |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 210 | // Setup required callbacks for the encoder and decoder and initialize them. |
brandtr | bdd555c | 2017-08-21 01:34:04 -0700 | [diff] [blame] | 211 | RTC_CHECK_EQ(encoder_->RegisterEncodeCompleteCallback(&encode_callback_), |
Åsa Persson | f0c4467 | 2017-10-24 16:03:39 +0200 | [diff] [blame] | 212 | WEBRTC_VIDEO_CODEC_OK); |
asapersson | 654d54c | 2017-02-10 00:16:07 -0800 | [diff] [blame] | 213 | |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 214 | // Initialize codecs so that they are ready to receive frames. |
Sergey Silkin | 1723cf9 | 2018-01-22 15:49:55 +0100 | [diff] [blame] | 215 | RTC_CHECK_EQ(encoder_->InitEncode(&config_.codec_settings, |
| 216 | static_cast<int>(config_.NumberOfCores()), |
| 217 | config_.max_payload_size_bytes), |
| 218 | WEBRTC_VIDEO_CODEC_OK); |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 219 | |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 220 | for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) { |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 221 | decode_callback_.push_back( |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 222 | rtc::MakeUnique<VideoProcessorDecodeCompleteCallback>(this, i)); |
| 223 | RTC_CHECK_EQ( |
| 224 | decoders_->at(i)->InitDecode(&config_.codec_settings, |
| 225 | static_cast<int>(config_.NumberOfCores())), |
| 226 | WEBRTC_VIDEO_CODEC_OK); |
| 227 | RTC_CHECK_EQ(decoders_->at(i)->RegisterDecodeCompleteCallback( |
| 228 | decode_callback_.at(i).get()), |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 229 | WEBRTC_VIDEO_CODEC_OK); |
| 230 | } |
kjellander@webrtc.org | 35a1756 | 2011-10-06 06:44:54 +0000 | [diff] [blame] | 231 | } |
| 232 | |
Åsa Persson | f0c4467 | 2017-10-24 16:03:39 +0200 | [diff] [blame] | 233 | VideoProcessor::~VideoProcessor() { |
brandtr | c8c5905 | 2017-08-21 06:44:16 -0700 | [diff] [blame] | 234 | RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); |
| 235 | |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 236 | // Explicitly reset codecs, in case they don't do that themselves when they |
| 237 | // go out of scope. |
brandtr | 77920a4 | 2017-08-11 07:48:15 -0700 | [diff] [blame] | 238 | RTC_CHECK_EQ(encoder_->Release(), WEBRTC_VIDEO_CODEC_OK); |
brandtr | bdd555c | 2017-08-21 01:34:04 -0700 | [diff] [blame] | 239 | encoder_->RegisterEncodeCompleteCallback(nullptr); |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 240 | for (auto& decoder : *decoders_) { |
| 241 | RTC_CHECK_EQ(decoder->Release(), WEBRTC_VIDEO_CODEC_OK); |
| 242 | decoder->RegisterDecodeCompleteCallback(nullptr); |
| 243 | } |
| 244 | |
Rasmus Brandt | d00c895 | 2018-03-14 12:29:57 +0100 | [diff] [blame] | 245 | // Sanity check. |
| 246 | RTC_CHECK_LE(input_frames_.size(), kMaxBufferedInputFrames); |
| 247 | |
| 248 | // Deal with manual memory management of EncodedImage's. |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 249 | for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) { |
| 250 | uint8_t* buffer = merged_encoded_frames_.at(i)._buffer; |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 251 | if (buffer) { |
| 252 | delete[] buffer; |
| 253 | } |
| 254 | } |
brandtr | 77920a4 | 2017-08-11 07:48:15 -0700 | [diff] [blame] | 255 | } |
| 256 | |
brandtr | 8935d97 | 2017-09-06 01:53:22 -0700 | [diff] [blame] | 257 | void VideoProcessor::ProcessFrame() { |
brandtr | c8c5905 | 2017-08-21 06:44:16 -0700 | [diff] [blame] | 258 | RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); |
Sergey Silkin | 3be2a55 | 2018-01-17 15:11:44 +0100 | [diff] [blame] | 259 | const size_t frame_number = last_inputed_frame_num_++; |
asapersson | 654d54c | 2017-02-10 00:16:07 -0800 | [diff] [blame] | 260 | |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 261 | // Get input frame and store for future quality calculation. |
| 262 | rtc::scoped_refptr<I420BufferInterface> buffer = |
| 263 | input_frame_reader_->ReadFrame(); |
brandtr | bdd555c | 2017-08-21 01:34:04 -0700 | [diff] [blame] | 264 | RTC_CHECK(buffer) << "Tried to read too many frames from the file."; |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 265 | const size_t timestamp = |
| 266 | last_inputed_timestamp_ + kVideoPayloadTypeFrequency / framerate_fps_; |
| 267 | VideoFrame input_frame(buffer, static_cast<uint32_t>(timestamp), |
| 268 | static_cast<int64_t>(timestamp / kMsToRtpTimestamp), |
| 269 | webrtc::kVideoRotation_0); |
Rasmus Brandt | d00c895 | 2018-03-14 12:29:57 +0100 | [diff] [blame] | 270 | // Store input frame as a reference for quality calculations. |
| 271 | if (config_.decode && !config_.measure_cpu) { |
| 272 | input_frames_.emplace(frame_number, input_frame); |
| 273 | } |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 274 | last_inputed_timestamp_ = timestamp; |
brandtr | 17b958c | 2017-03-07 01:41:43 -0800 | [diff] [blame] | 275 | |
Sergey Silkin | c89eed9 | 2018-04-01 23:57:51 +0200 | [diff] [blame] | 276 | post_encode_time_ns_ = 0; |
| 277 | |
Rasmus Brandt | 5f7a891 | 2018-02-28 17:17:15 +0100 | [diff] [blame] | 278 | // Create frame statistics object for all simulcast/spatial layers. |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 279 | for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) { |
| 280 | stats_->AddFrame(timestamp, i); |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 281 | } |
brandtr | 17b958c | 2017-03-07 01:41:43 -0800 | [diff] [blame] | 282 | |
| 283 | // For the highest measurement accuracy of the encode time, the start/stop |
| 284 | // time recordings should wrap the Encode call as tightly as possible. |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 285 | const int64_t encode_start_ns = rtc::TimeNanos(); |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 286 | for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) { |
| 287 | FrameStatistics* frame_stat = stats_->GetFrame(frame_number, i); |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 288 | frame_stat->encode_start_ns = encode_start_ns; |
| 289 | } |
| 290 | |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 291 | // Encode. |
| 292 | const std::vector<FrameType> frame_types = |
Kári Tristan Helgason | 169005d | 2018-05-22 13:34:14 +0200 | [diff] [blame] | 293 | FrameTypeForFrame(config_, frame_number); |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 294 | const int encode_return_code = |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 295 | encoder_->Encode(input_frame, nullptr, &frame_types); |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 296 | for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) { |
| 297 | FrameStatistics* frame_stat = stats_->GetFrame(frame_number, i); |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 298 | frame_stat->encode_return_code = encode_return_code; |
| 299 | } |
kjellander@webrtc.org | 35a1756 | 2011-10-06 06:44:54 +0000 | [diff] [blame] | 300 | } |
| 301 | |
Sergey Silkin | 3be2a55 | 2018-01-17 15:11:44 +0100 | [diff] [blame] | 302 | void VideoProcessor::SetRates(size_t bitrate_kbps, size_t framerate_fps) { |
brandtr | c8c5905 | 2017-08-21 06:44:16 -0700 | [diff] [blame] | 303 | RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 304 | framerate_fps_ = static_cast<uint32_t>(framerate_fps); |
Sergey Silkin | 3be2a55 | 2018-01-17 15:11:44 +0100 | [diff] [blame] | 305 | bitrate_allocation_ = bitrate_allocator_->GetAllocation( |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 306 | static_cast<uint32_t>(bitrate_kbps * 1000), framerate_fps_); |
| 307 | const int set_rates_result = |
| 308 | encoder_->SetRateAllocation(bitrate_allocation_, framerate_fps_); |
brandtr | bea36fd | 2017-08-07 03:36:54 -0700 | [diff] [blame] | 309 | RTC_DCHECK_GE(set_rates_result, 0) |
brandtr | bdd555c | 2017-08-21 01:34:04 -0700 | [diff] [blame] | 310 | << "Failed to update encoder with new rate " << bitrate_kbps << "."; |
brandtr | bea36fd | 2017-08-07 03:36:54 -0700 | [diff] [blame] | 311 | } |
| 312 | |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 313 | void VideoProcessor::FrameEncoded( |
| 314 | const webrtc::EncodedImage& encoded_image, |
| 315 | const webrtc::CodecSpecificInfo& codec_specific) { |
brandtr | c8c5905 | 2017-08-21 06:44:16 -0700 | [diff] [blame] | 316 | RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); |
| 317 | |
brandtr | 32e0d26 | 2017-02-15 05:29:38 -0800 | [diff] [blame] | 318 | // For the highest measurement accuracy of the encode time, the start/stop |
| 319 | // time recordings should wrap the Encode call as tightly as possible. |
Rasmus Brandt | 5f7a891 | 2018-02-28 17:17:15 +0100 | [diff] [blame] | 320 | const int64_t encode_stop_ns = rtc::TimeNanos(); |
brandtr | 32e0d26 | 2017-02-15 05:29:38 -0800 | [diff] [blame] | 321 | |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 322 | const VideoCodecType codec_type = codec_specific.codecType; |
Rasmus Brandt | f7a3558 | 2017-10-24 10:16:33 +0200 | [diff] [blame] | 323 | if (config_.encoded_frame_checker) { |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 324 | config_.encoded_frame_checker->CheckEncodedFrame(codec_type, encoded_image); |
Rasmus Brandt | f7a3558 | 2017-10-24 10:16:33 +0200 | [diff] [blame] | 325 | } |
brandtr | b78bc75 | 2017-02-22 01:26:59 -0800 | [diff] [blame] | 326 | |
Rasmus Brandt | d062a3c | 2018-03-08 16:45:54 +0100 | [diff] [blame] | 327 | // Layer metadata. |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 328 | size_t spatial_idx = 0; |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 329 | size_t temporal_idx = 0; |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 330 | GetLayerIndices(codec_specific, &spatial_idx, &temporal_idx); |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 331 | |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 332 | FrameStatistics* frame_stat = |
| 333 | stats_->GetFrameWithTimestamp(encoded_image._timeStamp, spatial_idx); |
Åsa Persson | a6e7b88 | 2018-01-19 14:57:10 +0100 | [diff] [blame] | 334 | const size_t frame_number = frame_stat->frame_number; |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 335 | |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 336 | // Ensure that the encode order is monotonically increasing, within this |
| 337 | // simulcast/spatial layer. |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 338 | RTC_CHECK(first_encoded_frame_[spatial_idx] || |
| 339 | last_encoded_frame_num_[spatial_idx] < frame_number); |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 340 | |
| 341 | // Ensure SVC spatial layers are delivered in ascending order. |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 342 | if (!first_encoded_frame_[spatial_idx] && |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 343 | config_.NumberOfSpatialLayers() > 1) { |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 344 | for (size_t i = 0; i < spatial_idx; ++i) { |
Sergey Silkin | 122ba6c | 2018-03-27 14:32:21 +0200 | [diff] [blame] | 345 | RTC_CHECK_LE(last_encoded_frame_num_[i], frame_number); |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 346 | } |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 347 | for (size_t i = spatial_idx + 1; i < num_simulcast_or_spatial_layers_; |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 348 | ++i) { |
| 349 | RTC_CHECK_GT(frame_number, last_encoded_frame_num_[i]); |
| 350 | } |
Sergey Silkin | 3be2a55 | 2018-01-17 15:11:44 +0100 | [diff] [blame] | 351 | } |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 352 | first_encoded_frame_[spatial_idx] = false; |
| 353 | last_encoded_frame_num_[spatial_idx] = frame_number; |
brandtr | 17b958c | 2017-03-07 01:41:43 -0800 | [diff] [blame] | 354 | |
brandtr | 8935d97 | 2017-09-06 01:53:22 -0700 | [diff] [blame] | 355 | // Update frame statistics. |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 356 | frame_stat->encoding_successful = true; |
Sergey Silkin | c89eed9 | 2018-04-01 23:57:51 +0200 | [diff] [blame] | 357 | frame_stat->encode_time_us = GetElapsedTimeMicroseconds( |
| 358 | frame_stat->encode_start_ns, encode_stop_ns - post_encode_time_ns_); |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 359 | frame_stat->target_bitrate_kbps = |
Sergey Silkin | bfd54ef | 2018-04-13 23:41:11 +0200 | [diff] [blame] | 360 | bitrate_allocation_.GetTemporalLayerSum(spatial_idx, temporal_idx) / 1000; |
Sergey Silkin | d4bc01b | 2018-03-09 14:31:24 +0100 | [diff] [blame] | 361 | frame_stat->length_bytes = encoded_image._length; |
brandtr | 17b958c | 2017-03-07 01:41:43 -0800 | [diff] [blame] | 362 | frame_stat->frame_type = encoded_image._frameType; |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 363 | frame_stat->temporal_idx = temporal_idx; |
| 364 | frame_stat->spatial_idx = spatial_idx; |
Sergey Silkin | 3be2a55 | 2018-01-17 15:11:44 +0100 | [diff] [blame] | 365 | frame_stat->max_nalu_size_bytes = GetMaxNaluSizeBytes(encoded_image, config_); |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 366 | frame_stat->qp = encoded_image.qp_; |
ssilkin | 612f858 | 2017-09-28 09:23:17 -0700 | [diff] [blame] | 367 | |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 368 | const size_t num_spatial_layers = config_.NumberOfSpatialLayers(); |
Sergey Silkin | bc0f0d3 | 2018-04-24 21:29:14 +0200 | [diff] [blame] | 369 | bool end_of_picture = false; |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 370 | if (codec_type == kVideoCodecVP9) { |
| 371 | const CodecSpecificInfoVP9& vp9_info = codec_specific.codecSpecific.VP9; |
| 372 | frame_stat->inter_layer_predicted = vp9_info.inter_layer_predicted; |
Sergey Silkin | 3c30c9c | 2018-05-02 09:18:48 +0200 | [diff] [blame] | 373 | frame_stat->non_ref_for_inter_layer_pred = |
| 374 | vp9_info.non_ref_for_inter_layer_pred; |
Sergey Silkin | bc0f0d3 | 2018-04-24 21:29:14 +0200 | [diff] [blame] | 375 | end_of_picture = vp9_info.end_of_picture; |
Sergey Silkin | 3c30c9c | 2018-05-02 09:18:48 +0200 | [diff] [blame] | 376 | } else { |
| 377 | frame_stat->inter_layer_predicted = false; |
| 378 | frame_stat->non_ref_for_inter_layer_pred = true; |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 379 | } |
| 380 | |
Sergey Silkin | 122ba6c | 2018-03-27 14:32:21 +0200 | [diff] [blame] | 381 | const webrtc::EncodedImage* encoded_image_for_decode = &encoded_image; |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 382 | if (config_.decode || encoded_frame_writers_) { |
| 383 | if (num_spatial_layers > 1) { |
| 384 | encoded_image_for_decode = BuildAndStoreSuperframe( |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 385 | encoded_image, codec_type, frame_number, spatial_idx, |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 386 | frame_stat->inter_layer_predicted); |
Rasmus Brandt | d00c895 | 2018-03-14 12:29:57 +0100 | [diff] [blame] | 387 | } |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 388 | } |
| 389 | |
| 390 | if (config_.decode) { |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 391 | DecodeFrame(*encoded_image_for_decode, spatial_idx); |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 392 | |
Sergey Silkin | 3c30c9c | 2018-05-02 09:18:48 +0200 | [diff] [blame] | 393 | if (end_of_picture && num_spatial_layers > 1) { |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 394 | // If inter-layer prediction is enabled and upper layer was dropped then |
| 395 | // base layer should be passed to upper layer decoder. Otherwise decoder |
| 396 | // won't be able to decode next superframe. |
| 397 | const EncodedImage* base_image = nullptr; |
Sergey Silkin | 3c30c9c | 2018-05-02 09:18:48 +0200 | [diff] [blame] | 398 | const FrameStatistics* base_stat = nullptr; |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 399 | for (size_t i = 0; i < num_spatial_layers; ++i) { |
Sergey Silkin | 3c30c9c | 2018-05-02 09:18:48 +0200 | [diff] [blame] | 400 | const bool layer_dropped = (first_decoded_frame_[i] || |
| 401 | last_decoded_frame_num_[i] < frame_number); |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 402 | |
| 403 | // Ensure current layer was decoded. |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 404 | RTC_CHECK(layer_dropped == false || i != spatial_idx); |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 405 | |
| 406 | if (!layer_dropped) { |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 407 | base_image = &merged_encoded_frames_[i]; |
Sergey Silkin | 3c30c9c | 2018-05-02 09:18:48 +0200 | [diff] [blame] | 408 | base_stat = |
| 409 | stats_->GetFrameWithTimestamp(encoded_image._timeStamp, i); |
| 410 | } else if (base_image && !base_stat->non_ref_for_inter_layer_pred) { |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 411 | DecodeFrame(*base_image, i); |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 412 | } |
| 413 | } |
| 414 | } |
Rasmus Brandt | d00c895 | 2018-03-14 12:29:57 +0100 | [diff] [blame] | 415 | } else { |
| 416 | frame_stat->decode_return_code = WEBRTC_VIDEO_CODEC_NO_OUTPUT; |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 417 | } |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 418 | |
| 419 | if (encoded_frame_writers_) { |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 420 | RTC_CHECK(encoded_frame_writers_->at(spatial_idx) |
Sergey Silkin | 122ba6c | 2018-03-27 14:32:21 +0200 | [diff] [blame] | 421 | ->WriteFrame(*encoded_image_for_decode, |
| 422 | config_.codec_settings.codecType)); |
brandtr | 8935d97 | 2017-09-06 01:53:22 -0700 | [diff] [blame] | 423 | } |
Sergey Silkin | c89eed9 | 2018-04-01 23:57:51 +0200 | [diff] [blame] | 424 | |
| 425 | if (!config_.IsAsyncCodec()) { |
| 426 | // To get pure encode time for next layers, measure time spent in encode |
| 427 | // callback and subtract it from encode time of next layers. |
| 428 | post_encode_time_ns_ += rtc::TimeNanos() - encode_stop_ns; |
| 429 | } |
kjellander@webrtc.org | 35a1756 | 2011-10-06 06:44:54 +0000 | [diff] [blame] | 430 | } |
| 431 | |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 432 | void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame, |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 433 | size_t spatial_idx) { |
brandtr | c8c5905 | 2017-08-21 06:44:16 -0700 | [diff] [blame] | 434 | RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); |
| 435 | |
brandtr | 32e0d26 | 2017-02-15 05:29:38 -0800 | [diff] [blame] | 436 | // For the highest measurement accuracy of the decode time, the start/stop |
| 437 | // time recordings should wrap the Decode call as tightly as possible. |
Rasmus Brandt | 5f7a891 | 2018-02-28 17:17:15 +0100 | [diff] [blame] | 438 | const int64_t decode_stop_ns = rtc::TimeNanos(); |
brandtr | 8bc9385 | 2017-02-15 05:19:51 -0800 | [diff] [blame] | 439 | |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 440 | FrameStatistics* frame_stat = |
| 441 | stats_->GetFrameWithTimestamp(decoded_frame.timestamp(), spatial_idx); |
Åsa Persson | a6e7b88 | 2018-01-19 14:57:10 +0100 | [diff] [blame] | 442 | const size_t frame_number = frame_stat->frame_number; |
Sergey Silkin | 64eaa99 | 2017-11-17 14:47:32 +0100 | [diff] [blame] | 443 | |
Sergey Silkin | 5613879 | 2018-05-02 10:50:55 +0200 | [diff] [blame] | 444 | if (decoded_frame_writers_ && !first_decoded_frame_[spatial_idx]) { |
| 445 | // Fill drops with last decoded frame to make them look like freeze at |
| 446 | // playback and to keep decoded layers in sync. |
| 447 | for (size_t i = last_decoded_frame_num_[spatial_idx] + 1; i < frame_number; |
| 448 | ++i) { |
| 449 | RTC_CHECK(decoded_frame_writers_->at(spatial_idx) |
| 450 | ->WriteFrame(decoded_frame_buffer_[spatial_idx].data())); |
| 451 | } |
| 452 | } |
| 453 | |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 454 | // Ensure that the decode order is monotonically increasing, within this |
| 455 | // simulcast/spatial layer. |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 456 | RTC_CHECK(first_decoded_frame_[spatial_idx] || |
| 457 | last_decoded_frame_num_[spatial_idx] < frame_number); |
| 458 | first_decoded_frame_[spatial_idx] = false; |
| 459 | last_decoded_frame_num_[spatial_idx] = frame_number; |
brandtr | 17b958c | 2017-03-07 01:41:43 -0800 | [diff] [blame] | 460 | |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 461 | // Update frame statistics. |
| 462 | frame_stat->decoding_successful = true; |
| 463 | frame_stat->decode_time_us = |
| 464 | GetElapsedTimeMicroseconds(frame_stat->decode_start_ns, decode_stop_ns); |
| 465 | frame_stat->decoded_width = decoded_frame.width(); |
| 466 | frame_stat->decoded_height = decoded_frame.height(); |
| 467 | |
Sergey Silkin | 64eaa99 | 2017-11-17 14:47:32 +0100 | [diff] [blame] | 468 | // Skip quality metrics calculation to not affect CPU usage. |
| 469 | if (!config_.measure_cpu) { |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 470 | const auto reference_frame = input_frames_.find(frame_number); |
| 471 | RTC_CHECK(reference_frame != input_frames_.cend()) |
| 472 | << "The codecs are either buffering too much, dropping too much, or " |
| 473 | "being too slow relative the input frame rate."; |
Sergey Silkin | 8d3758e | 2018-03-14 11:28:15 +0100 | [diff] [blame] | 474 | CalculateFrameQuality( |
| 475 | *reference_frame->second.video_frame_buffer()->ToI420(), |
| 476 | *decoded_frame.video_frame_buffer()->ToI420(), frame_stat); |
Niels Möller | 718a763 | 2016-06-13 13:06:01 +0200 | [diff] [blame] | 477 | |
Rasmus Brandt | d00c895 | 2018-03-14 12:29:57 +0100 | [diff] [blame] | 478 | // Erase all buffered input frames that we have moved past for all |
| 479 | // simulcast/spatial layers. Never buffer more than |
| 480 | // |kMaxBufferedInputFrames| frames, to protect against long runs of |
| 481 | // consecutive frame drops for a particular layer. |
| 482 | const auto min_last_decoded_frame_num = std::min_element( |
| 483 | last_decoded_frame_num_.cbegin(), last_decoded_frame_num_.cend()); |
| 484 | const size_t min_buffered_frame_num = std::max( |
| 485 | 0, static_cast<int>(frame_number) - kMaxBufferedInputFrames + 1); |
| 486 | RTC_CHECK(min_last_decoded_frame_num != last_decoded_frame_num_.cend()); |
| 487 | const auto input_frames_erase_before = input_frames_.lower_bound( |
| 488 | std::max(*min_last_decoded_frame_num, min_buffered_frame_num)); |
| 489 | input_frames_.erase(input_frames_.cbegin(), input_frames_erase_before); |
| 490 | } |
Sergey Silkin | 64eaa99 | 2017-11-17 14:47:32 +0100 | [diff] [blame] | 491 | |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 492 | if (decoded_frame_writers_) { |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 493 | ExtractI420BufferWithSize(decoded_frame, config_.codec_settings.width, |
Sergey Silkin | 5613879 | 2018-05-02 10:50:55 +0200 | [diff] [blame] | 494 | config_.codec_settings.height, |
| 495 | &decoded_frame_buffer_[spatial_idx]); |
| 496 | RTC_CHECK_EQ(decoded_frame_buffer_[spatial_idx].size(), |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 497 | decoded_frame_writers_->at(spatial_idx)->FrameLength()); |
| 498 | RTC_CHECK(decoded_frame_writers_->at(spatial_idx) |
Sergey Silkin | 5613879 | 2018-05-02 10:50:55 +0200 | [diff] [blame] | 499 | ->WriteFrame(decoded_frame_buffer_[spatial_idx].data())); |
Sergey Silkin | 64eaa99 | 2017-11-17 14:47:32 +0100 | [diff] [blame] | 500 | } |
Åsa Persson | f0c4467 | 2017-10-24 16:03:39 +0200 | [diff] [blame] | 501 | } |
brandtr | 17b958c | 2017-03-07 01:41:43 -0800 | [diff] [blame] | 502 | |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 503 | void VideoProcessor::DecodeFrame(const EncodedImage& encoded_image, |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 504 | size_t spatial_idx) { |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 505 | RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 506 | FrameStatistics* frame_stat = |
| 507 | stats_->GetFrameWithTimestamp(encoded_image._timeStamp, spatial_idx); |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 508 | |
| 509 | frame_stat->decode_start_ns = rtc::TimeNanos(); |
| 510 | frame_stat->decode_return_code = |
Niels Möller | 401d076 | 2018-05-08 11:54:29 +0200 | [diff] [blame] | 511 | decoders_->at(spatial_idx)->Decode(encoded_image, false, nullptr, 0); |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 512 | } |
| 513 | |
| 514 | const webrtc::EncodedImage* VideoProcessor::BuildAndStoreSuperframe( |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 515 | const EncodedImage& encoded_image, |
| 516 | const VideoCodecType codec, |
| 517 | size_t frame_number, |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 518 | size_t spatial_idx, |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 519 | bool inter_layer_predicted) { |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 520 | // Should only be called for SVC. |
| 521 | RTC_CHECK_GT(config_.NumberOfSpatialLayers(), 1); |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 522 | |
| 523 | EncodedImage base_image; |
| 524 | RTC_CHECK_EQ(base_image._length, 0); |
| 525 | |
Sergey Silkin | 122ba6c | 2018-03-27 14:32:21 +0200 | [diff] [blame] | 526 | // Each SVC layer is decoded with dedicated decoder. Find the nearest |
| 527 | // non-dropped base frame and merge it and current frame into superframe. |
Sergey Silkin | 645e2e0 | 2018-04-06 09:42:13 +0200 | [diff] [blame] | 528 | if (inter_layer_predicted) { |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 529 | for (int base_idx = static_cast<int>(spatial_idx) - 1; base_idx >= 0; |
Sergey Silkin | 122ba6c | 2018-03-27 14:32:21 +0200 | [diff] [blame] | 530 | --base_idx) { |
| 531 | EncodedImage lower_layer = merged_encoded_frames_.at(base_idx); |
| 532 | if (lower_layer._timeStamp == encoded_image._timeStamp) { |
| 533 | base_image = lower_layer; |
| 534 | break; |
| 535 | } |
| 536 | } |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 537 | } |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 538 | const size_t payload_size_bytes = base_image._length + encoded_image._length; |
| 539 | const size_t buffer_size_bytes = |
| 540 | payload_size_bytes + EncodedImage::GetBufferPaddingBytes(codec); |
| 541 | |
| 542 | uint8_t* copied_buffer = new uint8_t[buffer_size_bytes]; |
| 543 | RTC_CHECK(copied_buffer); |
| 544 | |
| 545 | if (base_image._length) { |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 546 | RTC_CHECK(base_image._buffer); |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 547 | memcpy(copied_buffer, base_image._buffer, base_image._length); |
| 548 | } |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 549 | memcpy(copied_buffer + base_image._length, encoded_image._buffer, |
| 550 | encoded_image._length); |
| 551 | |
| 552 | EncodedImage copied_image = encoded_image; |
| 553 | copied_image = encoded_image; |
| 554 | copied_image._buffer = copied_buffer; |
| 555 | copied_image._length = payload_size_bytes; |
| 556 | copied_image._size = buffer_size_bytes; |
| 557 | |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 558 | // Replace previous EncodedImage for this spatial layer. |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 559 | uint8_t* old_buffer = merged_encoded_frames_.at(spatial_idx)._buffer; |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 560 | if (old_buffer) { |
| 561 | delete[] old_buffer; |
| 562 | } |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 563 | merged_encoded_frames_.at(spatial_idx) = copied_image; |
Rasmus Brandt | 0f1c0bd | 2018-03-12 10:01:16 +0100 | [diff] [blame] | 564 | |
Sergey Silkin | bc20fe1 | 2018-04-15 21:06:16 +0200 | [diff] [blame] | 565 | return &merged_encoded_frames_.at(spatial_idx); |
Sergey Silkin | 10d9d59 | 2018-02-01 13:25:17 +0100 | [diff] [blame] | 566 | } |
| 567 | |
kjellander@webrtc.org | 35a1756 | 2011-10-06 06:44:54 +0000 | [diff] [blame] | 568 | } // namespace test |
| 569 | } // namespace webrtc |