Erik Språng | cc681cc | 2018-03-14 17:52:55 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame^] | 11 | #include <stdio.h> |
| 12 | #include <string.h> |
Erik Språng | cc681cc | 2018-03-14 17:52:55 +0100 | [diff] [blame] | 13 | #include <algorithm> |
| 14 | #include <string> |
| 15 | |
Karl Wiberg | 918f50c | 2018-07-05 11:40:33 +0200 | [diff] [blame] | 16 | #include "absl/memory/memory.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame^] | 17 | #include "absl/types/optional.h" |
| 18 | #include "api/video/i420_buffer.h" |
| 19 | #include "api/video/video_frame.h" |
| 20 | #include "api/video/video_frame_buffer.h" |
| 21 | #include "api/video/video_rotation.h" |
| 22 | #include "common_types.h" // NOLINT(build/include) |
| 23 | #include "modules/video_coding/codecs/vp8/include/vp8.h" |
Erik Språng | cc681cc | 2018-03-14 17:52:55 +0100 | [diff] [blame] | 24 | #include "modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame^] | 25 | #include "modules/video_coding/include/video_error_codes.h" |
Erik Språng | cc681cc | 2018-03-14 17:52:55 +0100 | [diff] [blame] | 26 | #include "rtc_base/checks.h" |
| 27 | #include "rtc_base/numerics/exp_filter.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame^] | 28 | #include "rtc_base/scoped_ref_ptr.h" |
Erik Språng | cc681cc | 2018-03-14 17:52:55 +0100 | [diff] [blame] | 29 | #include "rtc_base/timeutils.h" |
| 30 | #include "system_wrappers/include/field_trial.h" |
| 31 | #include "system_wrappers/include/metrics.h" |
| 32 | #include "third_party/libyuv/include/libyuv/convert.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame^] | 33 | #include "vpx/vp8.h" |
| 34 | #include "vpx/vp8dx.h" |
| 35 | #include "vpx/vpx_decoder.h" |
Erik Språng | cc681cc | 2018-03-14 17:52:55 +0100 | [diff] [blame] | 36 | |
| 37 | namespace webrtc { |
| 38 | namespace { |
| 39 | constexpr int kVp8ErrorPropagationTh = 30; |
| 40 | // vpx_decoder.h documentation indicates decode deadline is time in us, with |
| 41 | // "Set to zero for unlimited.", but actual implementation requires this to be |
| 42 | // a mode with 0 meaning allow delay and 1 not allowing it. |
| 43 | constexpr long kDecodeDeadlineRealtime = 1; // NOLINT |
| 44 | |
| 45 | const char kVp8PostProcArmFieldTrial[] = "WebRTC-VP8-Postproc-Config-Arm"; |
| 46 | |
| 47 | void GetPostProcParamsFromFieldTrialGroup( |
| 48 | LibvpxVp8Decoder::DeblockParams* deblock_params) { |
| 49 | std::string group = |
| 50 | webrtc::field_trial::FindFullName(kVp8PostProcArmFieldTrial); |
| 51 | if (group.empty()) |
| 52 | return; |
| 53 | |
| 54 | LibvpxVp8Decoder::DeblockParams params; |
| 55 | if (sscanf(group.c_str(), "Enabled-%d,%d,%d", ¶ms.max_level, |
| 56 | ¶ms.min_qp, ¶ms.degrade_qp) != 3) |
| 57 | return; |
| 58 | |
| 59 | if (params.max_level < 0 || params.max_level > 16) |
| 60 | return; |
| 61 | |
| 62 | if (params.min_qp < 0 || params.degrade_qp <= params.min_qp) |
| 63 | return; |
| 64 | |
| 65 | *deblock_params = params; |
| 66 | } |
| 67 | |
| 68 | } // namespace |
| 69 | |
Erik Språng | 5fbc0e0 | 2018-10-04 17:52:36 +0200 | [diff] [blame] | 70 | std::unique_ptr<VideoDecoder> VP8Decoder::Create() { |
Karl Wiberg | 918f50c | 2018-07-05 11:40:33 +0200 | [diff] [blame] | 71 | return absl::make_unique<LibvpxVp8Decoder>(); |
Erik Språng | cc681cc | 2018-03-14 17:52:55 +0100 | [diff] [blame] | 72 | } |
| 73 | |
| 74 | class LibvpxVp8Decoder::QpSmoother { |
| 75 | public: |
| 76 | QpSmoother() : last_sample_ms_(rtc::TimeMillis()), smoother_(kAlpha) {} |
| 77 | |
| 78 | int GetAvg() const { |
| 79 | float value = smoother_.filtered(); |
| 80 | return (value == rtc::ExpFilter::kValueUndefined) ? 0 |
| 81 | : static_cast<int>(value); |
| 82 | } |
| 83 | |
| 84 | void Add(float sample) { |
| 85 | int64_t now_ms = rtc::TimeMillis(); |
| 86 | smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample); |
| 87 | last_sample_ms_ = now_ms; |
| 88 | } |
| 89 | |
| 90 | void Reset() { smoother_.Reset(kAlpha); } |
| 91 | |
| 92 | private: |
| 93 | const float kAlpha = 0.95f; |
| 94 | int64_t last_sample_ms_; |
| 95 | rtc::ExpFilter smoother_; |
| 96 | }; |
| 97 | |
| 98 | LibvpxVp8Decoder::LibvpxVp8Decoder() |
| 99 | : use_postproc_arm_( |
| 100 | webrtc::field_trial::IsEnabled(kVp8PostProcArmFieldTrial)), |
| 101 | buffer_pool_(false, 300 /* max_number_of_buffers*/), |
| 102 | decode_complete_callback_(NULL), |
| 103 | inited_(false), |
| 104 | decoder_(NULL), |
| 105 | propagation_cnt_(-1), |
| 106 | last_frame_width_(0), |
| 107 | last_frame_height_(0), |
| 108 | key_frame_required_(true), |
| 109 | qp_smoother_(use_postproc_arm_ ? new QpSmoother() : nullptr) { |
| 110 | if (use_postproc_arm_) |
| 111 | GetPostProcParamsFromFieldTrialGroup(&deblock_); |
| 112 | } |
| 113 | |
| 114 | LibvpxVp8Decoder::~LibvpxVp8Decoder() { |
| 115 | inited_ = true; // in order to do the actual release |
| 116 | Release(); |
| 117 | } |
| 118 | |
| 119 | int LibvpxVp8Decoder::InitDecode(const VideoCodec* inst, int number_of_cores) { |
| 120 | int ret_val = Release(); |
| 121 | if (ret_val < 0) { |
| 122 | return ret_val; |
| 123 | } |
| 124 | if (decoder_ == NULL) { |
| 125 | decoder_ = new vpx_codec_ctx_t; |
| 126 | memset(decoder_, 0, sizeof(*decoder_)); |
| 127 | } |
| 128 | vpx_codec_dec_cfg_t cfg; |
| 129 | // Setting number of threads to a constant value (1) |
| 130 | cfg.threads = 1; |
| 131 | cfg.h = cfg.w = 0; // set after decode |
| 132 | |
| 133 | #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \ |
| 134 | defined(WEBRTC_ANDROID) |
| 135 | vpx_codec_flags_t flags = use_postproc_arm_ ? VPX_CODEC_USE_POSTPROC : 0; |
| 136 | #else |
| 137 | vpx_codec_flags_t flags = VPX_CODEC_USE_POSTPROC; |
| 138 | #endif |
| 139 | |
| 140 | if (vpx_codec_dec_init(decoder_, vpx_codec_vp8_dx(), &cfg, flags)) { |
| 141 | delete decoder_; |
| 142 | decoder_ = nullptr; |
| 143 | return WEBRTC_VIDEO_CODEC_MEMORY; |
| 144 | } |
| 145 | |
| 146 | propagation_cnt_ = -1; |
| 147 | inited_ = true; |
| 148 | |
| 149 | // Always start with a complete key frame. |
| 150 | key_frame_required_ = true; |
| 151 | return WEBRTC_VIDEO_CODEC_OK; |
| 152 | } |
| 153 | |
| 154 | int LibvpxVp8Decoder::Decode(const EncodedImage& input_image, |
| 155 | bool missing_frames, |
Erik Språng | cc681cc | 2018-03-14 17:52:55 +0100 | [diff] [blame] | 156 | const CodecSpecificInfo* codec_specific_info, |
| 157 | int64_t /*render_time_ms*/) { |
| 158 | if (!inited_) { |
| 159 | return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 160 | } |
| 161 | if (decode_complete_callback_ == NULL) { |
| 162 | return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 163 | } |
| 164 | if (input_image._buffer == NULL && input_image._length > 0) { |
| 165 | // Reset to avoid requesting key frames too often. |
| 166 | if (propagation_cnt_ > 0) |
| 167 | propagation_cnt_ = 0; |
| 168 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 169 | } |
| 170 | |
| 171 | // Post process configurations. |
| 172 | #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \ |
| 173 | defined(WEBRTC_ANDROID) |
| 174 | if (use_postproc_arm_) { |
| 175 | vp8_postproc_cfg_t ppcfg; |
| 176 | ppcfg.post_proc_flag = VP8_MFQE; |
| 177 | // For low resolutions, use stronger deblocking filter. |
| 178 | int last_width_x_height = last_frame_width_ * last_frame_height_; |
| 179 | if (last_width_x_height > 0 && last_width_x_height <= 320 * 240) { |
| 180 | // Enable the deblock and demacroblocker based on qp thresholds. |
| 181 | RTC_DCHECK(qp_smoother_); |
| 182 | int qp = qp_smoother_->GetAvg(); |
| 183 | if (qp > deblock_.min_qp) { |
| 184 | int level = deblock_.max_level; |
| 185 | if (qp < deblock_.degrade_qp) { |
| 186 | // Use lower level. |
| 187 | level = deblock_.max_level * (qp - deblock_.min_qp) / |
| 188 | (deblock_.degrade_qp - deblock_.min_qp); |
| 189 | } |
| 190 | // Deblocking level only affects VP8_DEMACROBLOCK. |
| 191 | ppcfg.deblocking_level = std::max(level, 1); |
| 192 | ppcfg.post_proc_flag |= VP8_DEBLOCK | VP8_DEMACROBLOCK; |
| 193 | } |
| 194 | } |
| 195 | vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg); |
| 196 | } |
| 197 | #else |
| 198 | vp8_postproc_cfg_t ppcfg; |
| 199 | // MFQE enabled to reduce key frame popping. |
| 200 | ppcfg.post_proc_flag = VP8_MFQE | VP8_DEBLOCK; |
| 201 | // For VGA resolutions and lower, enable the demacroblocker postproc. |
| 202 | if (last_frame_width_ * last_frame_height_ <= 640 * 360) { |
| 203 | ppcfg.post_proc_flag |= VP8_DEMACROBLOCK; |
| 204 | } |
| 205 | // Strength of deblocking filter. Valid range:[0,16] |
| 206 | ppcfg.deblocking_level = 3; |
| 207 | vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg); |
| 208 | #endif |
| 209 | |
| 210 | // Always start with a complete key frame. |
| 211 | if (key_frame_required_) { |
| 212 | if (input_image._frameType != kVideoFrameKey) |
| 213 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 214 | // We have a key frame - is it complete? |
| 215 | if (input_image._completeFrame) { |
| 216 | key_frame_required_ = false; |
| 217 | } else { |
| 218 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 219 | } |
| 220 | } |
| 221 | // Restrict error propagation using key frame requests. |
| 222 | // Reset on a key frame refresh. |
| 223 | if (input_image._frameType == kVideoFrameKey && input_image._completeFrame) { |
| 224 | propagation_cnt_ = -1; |
| 225 | // Start count on first loss. |
| 226 | } else if ((!input_image._completeFrame || missing_frames) && |
| 227 | propagation_cnt_ == -1) { |
| 228 | propagation_cnt_ = 0; |
| 229 | } |
| 230 | if (propagation_cnt_ >= 0) { |
| 231 | propagation_cnt_++; |
| 232 | } |
| 233 | |
| 234 | vpx_codec_iter_t iter = NULL; |
| 235 | vpx_image_t* img; |
| 236 | int ret; |
| 237 | |
| 238 | // Check for missing frames. |
| 239 | if (missing_frames) { |
| 240 | // Call decoder with zero data length to signal missing frames. |
| 241 | if (vpx_codec_decode(decoder_, NULL, 0, 0, kDecodeDeadlineRealtime)) { |
| 242 | // Reset to avoid requesting key frames too often. |
| 243 | if (propagation_cnt_ > 0) |
| 244 | propagation_cnt_ = 0; |
| 245 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 246 | } |
| 247 | img = vpx_codec_get_frame(decoder_, &iter); |
| 248 | iter = NULL; |
| 249 | } |
| 250 | |
| 251 | uint8_t* buffer = input_image._buffer; |
| 252 | if (input_image._length == 0) { |
| 253 | buffer = NULL; // Triggers full frame concealment. |
| 254 | } |
| 255 | if (vpx_codec_decode(decoder_, buffer, input_image._length, 0, |
| 256 | kDecodeDeadlineRealtime)) { |
| 257 | // Reset to avoid requesting key frames too often. |
| 258 | if (propagation_cnt_ > 0) { |
| 259 | propagation_cnt_ = 0; |
| 260 | } |
| 261 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 262 | } |
| 263 | |
| 264 | img = vpx_codec_get_frame(decoder_, &iter); |
| 265 | int qp; |
| 266 | vpx_codec_err_t vpx_ret = |
| 267 | vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp); |
| 268 | RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK); |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 269 | ret = ReturnFrame(img, input_image.Timestamp(), input_image.ntp_time_ms_, qp); |
Erik Språng | cc681cc | 2018-03-14 17:52:55 +0100 | [diff] [blame] | 270 | if (ret != 0) { |
| 271 | // Reset to avoid requesting key frames too often. |
| 272 | if (ret < 0 && propagation_cnt_ > 0) |
| 273 | propagation_cnt_ = 0; |
| 274 | return ret; |
| 275 | } |
| 276 | // Check Vs. threshold |
| 277 | if (propagation_cnt_ > kVp8ErrorPropagationTh) { |
| 278 | // Reset to avoid requesting key frames too often. |
| 279 | propagation_cnt_ = 0; |
| 280 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 281 | } |
| 282 | return WEBRTC_VIDEO_CODEC_OK; |
| 283 | } |
| 284 | |
| 285 | int LibvpxVp8Decoder::ReturnFrame(const vpx_image_t* img, |
| 286 | uint32_t timestamp, |
| 287 | int64_t ntp_time_ms, |
| 288 | int qp) { |
| 289 | if (img == NULL) { |
| 290 | // Decoder OK and NULL image => No show frame |
| 291 | return WEBRTC_VIDEO_CODEC_NO_OUTPUT; |
| 292 | } |
| 293 | if (qp_smoother_) { |
| 294 | if (last_frame_width_ != static_cast<int>(img->d_w) || |
| 295 | last_frame_height_ != static_cast<int>(img->d_h)) { |
| 296 | qp_smoother_->Reset(); |
| 297 | } |
| 298 | qp_smoother_->Add(qp); |
| 299 | } |
| 300 | last_frame_width_ = img->d_w; |
| 301 | last_frame_height_ = img->d_h; |
| 302 | // Allocate memory for decoded image. |
| 303 | rtc::scoped_refptr<I420Buffer> buffer = |
| 304 | buffer_pool_.CreateBuffer(img->d_w, img->d_h); |
| 305 | if (!buffer.get()) { |
| 306 | // Pool has too many pending frames. |
| 307 | RTC_HISTOGRAM_BOOLEAN("WebRTC.Video.LibvpxVp8Decoder.TooManyPendingFrames", |
| 308 | 1); |
| 309 | return WEBRTC_VIDEO_CODEC_NO_OUTPUT; |
| 310 | } |
| 311 | |
| 312 | libyuv::I420Copy(img->planes[VPX_PLANE_Y], img->stride[VPX_PLANE_Y], |
| 313 | img->planes[VPX_PLANE_U], img->stride[VPX_PLANE_U], |
| 314 | img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V], |
| 315 | buffer->MutableDataY(), buffer->StrideY(), |
| 316 | buffer->MutableDataU(), buffer->StrideU(), |
| 317 | buffer->MutableDataV(), buffer->StrideV(), img->d_w, |
| 318 | img->d_h); |
| 319 | |
Emircan Uysaler | ff52e88 | 2018-08-03 23:03:01 +0000 | [diff] [blame] | 320 | VideoFrame decoded_image(buffer, timestamp, 0, kVideoRotation_0); |
| 321 | decoded_image.set_ntp_time_ms(ntp_time_ms); |
Danil Chapovalov | 0040b66 | 2018-06-18 10:48:16 +0200 | [diff] [blame] | 322 | decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp); |
Erik Språng | cc681cc | 2018-03-14 17:52:55 +0100 | [diff] [blame] | 323 | |
| 324 | return WEBRTC_VIDEO_CODEC_OK; |
| 325 | } |
| 326 | |
| 327 | int LibvpxVp8Decoder::RegisterDecodeCompleteCallback( |
| 328 | DecodedImageCallback* callback) { |
| 329 | decode_complete_callback_ = callback; |
| 330 | return WEBRTC_VIDEO_CODEC_OK; |
| 331 | } |
| 332 | |
| 333 | int LibvpxVp8Decoder::Release() { |
| 334 | int ret_val = WEBRTC_VIDEO_CODEC_OK; |
| 335 | |
| 336 | if (decoder_ != NULL) { |
| 337 | if (inited_) { |
| 338 | if (vpx_codec_destroy(decoder_)) { |
| 339 | ret_val = WEBRTC_VIDEO_CODEC_MEMORY; |
| 340 | } |
| 341 | } |
| 342 | delete decoder_; |
| 343 | decoder_ = NULL; |
| 344 | } |
| 345 | buffer_pool_.Release(); |
| 346 | inited_ = false; |
| 347 | return ret_val; |
| 348 | } |
| 349 | |
| 350 | const char* LibvpxVp8Decoder::ImplementationName() const { |
| 351 | return "libvpx"; |
| 352 | } |
| 353 | } // namespace webrtc |