Erik Språng | cc681cc | 2018-03-14 17:52:55 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Erik Språng | cc681cc | 2018-03-14 17:52:55 +0100 | [diff] [blame] | 11 | #include <algorithm> |
| 12 | #include <string> |
| 13 | |
| 14 | #include "common_video/libyuv/include/webrtc_libyuv.h" |
| 15 | #include "modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h" |
| 16 | #include "rtc_base/checks.h" |
| 17 | #include "rtc_base/numerics/exp_filter.h" |
| 18 | #include "rtc_base/ptr_util.h" |
| 19 | #include "rtc_base/timeutils.h" |
| 20 | #include "system_wrappers/include/field_trial.h" |
| 21 | #include "system_wrappers/include/metrics.h" |
| 22 | #include "third_party/libyuv/include/libyuv/convert.h" |
| 23 | #include "third_party/libyuv/include/libyuv/scale.h" |
| 24 | |
| 25 | namespace webrtc { |
| 26 | namespace { |
| 27 | constexpr int kVp8ErrorPropagationTh = 30; |
| 28 | // vpx_decoder.h documentation indicates decode deadline is time in us, with |
| 29 | // "Set to zero for unlimited.", but actual implementation requires this to be |
| 30 | // a mode with 0 meaning allow delay and 1 not allowing it. |
| 31 | constexpr long kDecodeDeadlineRealtime = 1; // NOLINT |
| 32 | |
| 33 | const char kVp8PostProcArmFieldTrial[] = "WebRTC-VP8-Postproc-Config-Arm"; |
| 34 | |
| 35 | void GetPostProcParamsFromFieldTrialGroup( |
| 36 | LibvpxVp8Decoder::DeblockParams* deblock_params) { |
| 37 | std::string group = |
| 38 | webrtc::field_trial::FindFullName(kVp8PostProcArmFieldTrial); |
| 39 | if (group.empty()) |
| 40 | return; |
| 41 | |
| 42 | LibvpxVp8Decoder::DeblockParams params; |
| 43 | if (sscanf(group.c_str(), "Enabled-%d,%d,%d", ¶ms.max_level, |
| 44 | ¶ms.min_qp, ¶ms.degrade_qp) != 3) |
| 45 | return; |
| 46 | |
| 47 | if (params.max_level < 0 || params.max_level > 16) |
| 48 | return; |
| 49 | |
| 50 | if (params.min_qp < 0 || params.degrade_qp <= params.min_qp) |
| 51 | return; |
| 52 | |
| 53 | *deblock_params = params; |
| 54 | } |
| 55 | |
| 56 | } // namespace |
| 57 | |
| 58 | std::unique_ptr<VP8Decoder> VP8Decoder::Create() { |
| 59 | return rtc::MakeUnique<LibvpxVp8Decoder>(); |
| 60 | } |
| 61 | |
| 62 | class LibvpxVp8Decoder::QpSmoother { |
| 63 | public: |
| 64 | QpSmoother() : last_sample_ms_(rtc::TimeMillis()), smoother_(kAlpha) {} |
| 65 | |
| 66 | int GetAvg() const { |
| 67 | float value = smoother_.filtered(); |
| 68 | return (value == rtc::ExpFilter::kValueUndefined) ? 0 |
| 69 | : static_cast<int>(value); |
| 70 | } |
| 71 | |
| 72 | void Add(float sample) { |
| 73 | int64_t now_ms = rtc::TimeMillis(); |
| 74 | smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample); |
| 75 | last_sample_ms_ = now_ms; |
| 76 | } |
| 77 | |
| 78 | void Reset() { smoother_.Reset(kAlpha); } |
| 79 | |
| 80 | private: |
| 81 | const float kAlpha = 0.95f; |
| 82 | int64_t last_sample_ms_; |
| 83 | rtc::ExpFilter smoother_; |
| 84 | }; |
| 85 | |
| 86 | LibvpxVp8Decoder::LibvpxVp8Decoder() |
| 87 | : use_postproc_arm_( |
| 88 | webrtc::field_trial::IsEnabled(kVp8PostProcArmFieldTrial)), |
| 89 | buffer_pool_(false, 300 /* max_number_of_buffers*/), |
| 90 | decode_complete_callback_(NULL), |
| 91 | inited_(false), |
| 92 | decoder_(NULL), |
| 93 | propagation_cnt_(-1), |
| 94 | last_frame_width_(0), |
| 95 | last_frame_height_(0), |
| 96 | key_frame_required_(true), |
| 97 | qp_smoother_(use_postproc_arm_ ? new QpSmoother() : nullptr) { |
| 98 | if (use_postproc_arm_) |
| 99 | GetPostProcParamsFromFieldTrialGroup(&deblock_); |
| 100 | } |
| 101 | |
| 102 | LibvpxVp8Decoder::~LibvpxVp8Decoder() { |
| 103 | inited_ = true; // in order to do the actual release |
| 104 | Release(); |
| 105 | } |
| 106 | |
| 107 | int LibvpxVp8Decoder::InitDecode(const VideoCodec* inst, int number_of_cores) { |
| 108 | int ret_val = Release(); |
| 109 | if (ret_val < 0) { |
| 110 | return ret_val; |
| 111 | } |
| 112 | if (decoder_ == NULL) { |
| 113 | decoder_ = new vpx_codec_ctx_t; |
| 114 | memset(decoder_, 0, sizeof(*decoder_)); |
| 115 | } |
| 116 | vpx_codec_dec_cfg_t cfg; |
| 117 | // Setting number of threads to a constant value (1) |
| 118 | cfg.threads = 1; |
| 119 | cfg.h = cfg.w = 0; // set after decode |
| 120 | |
| 121 | #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \ |
| 122 | defined(WEBRTC_ANDROID) |
| 123 | vpx_codec_flags_t flags = use_postproc_arm_ ? VPX_CODEC_USE_POSTPROC : 0; |
| 124 | #else |
| 125 | vpx_codec_flags_t flags = VPX_CODEC_USE_POSTPROC; |
| 126 | #endif |
| 127 | |
| 128 | if (vpx_codec_dec_init(decoder_, vpx_codec_vp8_dx(), &cfg, flags)) { |
| 129 | delete decoder_; |
| 130 | decoder_ = nullptr; |
| 131 | return WEBRTC_VIDEO_CODEC_MEMORY; |
| 132 | } |
| 133 | |
| 134 | propagation_cnt_ = -1; |
| 135 | inited_ = true; |
| 136 | |
| 137 | // Always start with a complete key frame. |
| 138 | key_frame_required_ = true; |
| 139 | return WEBRTC_VIDEO_CODEC_OK; |
| 140 | } |
| 141 | |
| 142 | int LibvpxVp8Decoder::Decode(const EncodedImage& input_image, |
| 143 | bool missing_frames, |
Erik Språng | cc681cc | 2018-03-14 17:52:55 +0100 | [diff] [blame] | 144 | const CodecSpecificInfo* codec_specific_info, |
| 145 | int64_t /*render_time_ms*/) { |
| 146 | if (!inited_) { |
| 147 | return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 148 | } |
| 149 | if (decode_complete_callback_ == NULL) { |
| 150 | return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 151 | } |
| 152 | if (input_image._buffer == NULL && input_image._length > 0) { |
| 153 | // Reset to avoid requesting key frames too often. |
| 154 | if (propagation_cnt_ > 0) |
| 155 | propagation_cnt_ = 0; |
| 156 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 157 | } |
| 158 | |
| 159 | // Post process configurations. |
| 160 | #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \ |
| 161 | defined(WEBRTC_ANDROID) |
| 162 | if (use_postproc_arm_) { |
| 163 | vp8_postproc_cfg_t ppcfg; |
| 164 | ppcfg.post_proc_flag = VP8_MFQE; |
| 165 | // For low resolutions, use stronger deblocking filter. |
| 166 | int last_width_x_height = last_frame_width_ * last_frame_height_; |
| 167 | if (last_width_x_height > 0 && last_width_x_height <= 320 * 240) { |
| 168 | // Enable the deblock and demacroblocker based on qp thresholds. |
| 169 | RTC_DCHECK(qp_smoother_); |
| 170 | int qp = qp_smoother_->GetAvg(); |
| 171 | if (qp > deblock_.min_qp) { |
| 172 | int level = deblock_.max_level; |
| 173 | if (qp < deblock_.degrade_qp) { |
| 174 | // Use lower level. |
| 175 | level = deblock_.max_level * (qp - deblock_.min_qp) / |
| 176 | (deblock_.degrade_qp - deblock_.min_qp); |
| 177 | } |
| 178 | // Deblocking level only affects VP8_DEMACROBLOCK. |
| 179 | ppcfg.deblocking_level = std::max(level, 1); |
| 180 | ppcfg.post_proc_flag |= VP8_DEBLOCK | VP8_DEMACROBLOCK; |
| 181 | } |
| 182 | } |
| 183 | vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg); |
| 184 | } |
| 185 | #else |
| 186 | vp8_postproc_cfg_t ppcfg; |
| 187 | // MFQE enabled to reduce key frame popping. |
| 188 | ppcfg.post_proc_flag = VP8_MFQE | VP8_DEBLOCK; |
| 189 | // For VGA resolutions and lower, enable the demacroblocker postproc. |
| 190 | if (last_frame_width_ * last_frame_height_ <= 640 * 360) { |
| 191 | ppcfg.post_proc_flag |= VP8_DEMACROBLOCK; |
| 192 | } |
| 193 | // Strength of deblocking filter. Valid range:[0,16] |
| 194 | ppcfg.deblocking_level = 3; |
| 195 | vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg); |
| 196 | #endif |
| 197 | |
| 198 | // Always start with a complete key frame. |
| 199 | if (key_frame_required_) { |
| 200 | if (input_image._frameType != kVideoFrameKey) |
| 201 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 202 | // We have a key frame - is it complete? |
| 203 | if (input_image._completeFrame) { |
| 204 | key_frame_required_ = false; |
| 205 | } else { |
| 206 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 207 | } |
| 208 | } |
| 209 | // Restrict error propagation using key frame requests. |
| 210 | // Reset on a key frame refresh. |
| 211 | if (input_image._frameType == kVideoFrameKey && input_image._completeFrame) { |
| 212 | propagation_cnt_ = -1; |
| 213 | // Start count on first loss. |
| 214 | } else if ((!input_image._completeFrame || missing_frames) && |
| 215 | propagation_cnt_ == -1) { |
| 216 | propagation_cnt_ = 0; |
| 217 | } |
| 218 | if (propagation_cnt_ >= 0) { |
| 219 | propagation_cnt_++; |
| 220 | } |
| 221 | |
| 222 | vpx_codec_iter_t iter = NULL; |
| 223 | vpx_image_t* img; |
| 224 | int ret; |
| 225 | |
| 226 | // Check for missing frames. |
| 227 | if (missing_frames) { |
| 228 | // Call decoder with zero data length to signal missing frames. |
| 229 | if (vpx_codec_decode(decoder_, NULL, 0, 0, kDecodeDeadlineRealtime)) { |
| 230 | // Reset to avoid requesting key frames too often. |
| 231 | if (propagation_cnt_ > 0) |
| 232 | propagation_cnt_ = 0; |
| 233 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 234 | } |
| 235 | img = vpx_codec_get_frame(decoder_, &iter); |
| 236 | iter = NULL; |
| 237 | } |
| 238 | |
| 239 | uint8_t* buffer = input_image._buffer; |
| 240 | if (input_image._length == 0) { |
| 241 | buffer = NULL; // Triggers full frame concealment. |
| 242 | } |
| 243 | if (vpx_codec_decode(decoder_, buffer, input_image._length, 0, |
| 244 | kDecodeDeadlineRealtime)) { |
| 245 | // Reset to avoid requesting key frames too often. |
| 246 | if (propagation_cnt_ > 0) { |
| 247 | propagation_cnt_ = 0; |
| 248 | } |
| 249 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 250 | } |
| 251 | |
| 252 | img = vpx_codec_get_frame(decoder_, &iter); |
| 253 | int qp; |
| 254 | vpx_codec_err_t vpx_ret = |
| 255 | vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp); |
| 256 | RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK); |
| 257 | ret = ReturnFrame(img, input_image._timeStamp, input_image.ntp_time_ms_, qp); |
| 258 | if (ret != 0) { |
| 259 | // Reset to avoid requesting key frames too often. |
| 260 | if (ret < 0 && propagation_cnt_ > 0) |
| 261 | propagation_cnt_ = 0; |
| 262 | return ret; |
| 263 | } |
| 264 | // Check Vs. threshold |
| 265 | if (propagation_cnt_ > kVp8ErrorPropagationTh) { |
| 266 | // Reset to avoid requesting key frames too often. |
| 267 | propagation_cnt_ = 0; |
| 268 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 269 | } |
| 270 | return WEBRTC_VIDEO_CODEC_OK; |
| 271 | } |
| 272 | |
| 273 | int LibvpxVp8Decoder::ReturnFrame(const vpx_image_t* img, |
| 274 | uint32_t timestamp, |
| 275 | int64_t ntp_time_ms, |
| 276 | int qp) { |
| 277 | if (img == NULL) { |
| 278 | // Decoder OK and NULL image => No show frame |
| 279 | return WEBRTC_VIDEO_CODEC_NO_OUTPUT; |
| 280 | } |
| 281 | if (qp_smoother_) { |
| 282 | if (last_frame_width_ != static_cast<int>(img->d_w) || |
| 283 | last_frame_height_ != static_cast<int>(img->d_h)) { |
| 284 | qp_smoother_->Reset(); |
| 285 | } |
| 286 | qp_smoother_->Add(qp); |
| 287 | } |
| 288 | last_frame_width_ = img->d_w; |
| 289 | last_frame_height_ = img->d_h; |
| 290 | // Allocate memory for decoded image. |
| 291 | rtc::scoped_refptr<I420Buffer> buffer = |
| 292 | buffer_pool_.CreateBuffer(img->d_w, img->d_h); |
| 293 | if (!buffer.get()) { |
| 294 | // Pool has too many pending frames. |
| 295 | RTC_HISTOGRAM_BOOLEAN("WebRTC.Video.LibvpxVp8Decoder.TooManyPendingFrames", |
| 296 | 1); |
| 297 | return WEBRTC_VIDEO_CODEC_NO_OUTPUT; |
| 298 | } |
| 299 | |
| 300 | libyuv::I420Copy(img->planes[VPX_PLANE_Y], img->stride[VPX_PLANE_Y], |
| 301 | img->planes[VPX_PLANE_U], img->stride[VPX_PLANE_U], |
| 302 | img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V], |
| 303 | buffer->MutableDataY(), buffer->StrideY(), |
| 304 | buffer->MutableDataU(), buffer->StrideU(), |
| 305 | buffer->MutableDataV(), buffer->StrideV(), img->d_w, |
| 306 | img->d_h); |
| 307 | |
| 308 | VideoFrame decoded_image(buffer, timestamp, 0, kVideoRotation_0); |
| 309 | decoded_image.set_ntp_time_ms(ntp_time_ms); |
Danil Chapovalov | 0040b66 | 2018-06-18 10:48:16 +0200 | [diff] [blame] | 310 | decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp); |
Erik Språng | cc681cc | 2018-03-14 17:52:55 +0100 | [diff] [blame] | 311 | |
| 312 | return WEBRTC_VIDEO_CODEC_OK; |
| 313 | } |
| 314 | |
| 315 | int LibvpxVp8Decoder::RegisterDecodeCompleteCallback( |
| 316 | DecodedImageCallback* callback) { |
| 317 | decode_complete_callback_ = callback; |
| 318 | return WEBRTC_VIDEO_CODEC_OK; |
| 319 | } |
| 320 | |
| 321 | int LibvpxVp8Decoder::Release() { |
| 322 | int ret_val = WEBRTC_VIDEO_CODEC_OK; |
| 323 | |
| 324 | if (decoder_ != NULL) { |
| 325 | if (inited_) { |
| 326 | if (vpx_codec_destroy(decoder_)) { |
| 327 | ret_val = WEBRTC_VIDEO_CODEC_MEMORY; |
| 328 | } |
| 329 | } |
| 330 | delete decoder_; |
| 331 | decoder_ = NULL; |
| 332 | } |
| 333 | buffer_pool_.Release(); |
| 334 | inited_ = false; |
| 335 | return ret_val; |
| 336 | } |
| 337 | |
| 338 | const char* LibvpxVp8Decoder::ImplementationName() const { |
| 339 | return "libvpx"; |
| 340 | } |
| 341 | } // namespace webrtc |