Marat Dukhan | 062bee3 | 2021-05-27 20:31:07 -0700 | [diff] [blame^] | 1 | // Copyright (c) Facebook, Inc. and its affiliates. |
| 2 | // All rights reserved. |
| 3 | // |
| 4 | // Copyright 2019 Google LLC |
| 5 | // |
| 6 | // This source code is licensed under the BSD-style license found in the |
| 7 | // LICENSE file in the root directory of this source tree. |
| 8 | |
| 9 | #include <assert.h> |
| 10 | #include <stdint.h> |
| 11 | #include <stddef.h> |
| 12 | |
| 13 | #include <fp16/bitcasts.h> |
| 14 | |
| 15 | #include <xnnpack/math.h> |
| 16 | #include <xnnpack/requantization-stubs.h> |
| 17 | |
| 18 | |
| 19 | void xnn_qs8_requantize_rndnu__scalar( |
| 20 | size_t n, |
| 21 | const int32_t* input, |
| 22 | float scale, |
| 23 | int8_t zero_point, |
| 24 | int8_t qmin, |
| 25 | int8_t qmax, |
| 26 | int8_t* output) |
| 27 | { |
| 28 | assert(n % 4 == 0); |
| 29 | assert(scale < 1.0f); |
| 30 | assert(scale >= 0x1.0p-32f); |
| 31 | |
| 32 | const uint32_t scale_bits = fp32_to_bits(scale); |
| 33 | const int32_t multiplier = ((int32_t) scale_bits & INT32_C(0x007FFFFF)) | INT32_C(0x00800000); |
| 34 | const uint32_t shift = 127 + 23 - (scale_bits >> 23); |
| 35 | assert(shift >= 24); |
| 36 | assert(shift < 56); |
| 37 | |
| 38 | const int64_t rounding = INT64_C(1) << (shift - 1); |
| 39 | const int32_t smin = (int32_t) qmin - (int32_t) zero_point; |
| 40 | const int32_t smax = (int32_t) qmax - (int32_t) zero_point; |
| 41 | for (; n != 0; n -= 4) { |
| 42 | const int32_t x = input[0]; |
| 43 | const int32_t y = input[1]; |
| 44 | const int32_t z = input[2]; |
| 45 | const int32_t w = input[3]; |
| 46 | input += 4; |
| 47 | |
| 48 | // Compute full 64-bit product of signed 32-bit factors. |
| 49 | // |
| 50 | // Note: multiplier can be treated as either signed or unsigned. |
| 51 | const int64_t x_product = (int64_t) x * (int64_t) multiplier; |
| 52 | const int64_t y_product = (int64_t) y * (int64_t) multiplier; |
| 53 | const int64_t z_product = (int64_t) z * (int64_t) multiplier; |
| 54 | const int64_t w_product = (int64_t) w * (int64_t) multiplier; |
| 55 | |
| 56 | // Arithmetically shift the full 64-bit product right with rounding. |
| 57 | // Rounding is performed towards closest integer, with midpoints rounded up. |
| 58 | // |
| 59 | // Note that although rounding is precomputed, it is dependent on shift value, and on processors with 64-bit |
| 60 | // "right shift with rounding" instruction each line below can be represented by just one such instruction |
| 61 | // (e.g. VRSHL.S64 on ARM NEON, SRSHL in ARM64 Advanced SIMD). |
| 62 | const int32_t x_scaled = (int32_t) asr_s64(x_product + rounding, shift); |
| 63 | const int32_t y_scaled = (int32_t) asr_s64(y_product + rounding, shift); |
| 64 | const int32_t z_scaled = (int32_t) asr_s64(z_product + rounding, shift); |
| 65 | const int32_t w_scaled = (int32_t) asr_s64(w_product + rounding, shift); |
| 66 | |
| 67 | // Clamp scaled value with zero point between (qmin - zero point) and (qmax - zero point). |
| 68 | const int32_t x_clamped = math_min_s32(math_max_s32(x_scaled, smin), smax); |
| 69 | const int32_t y_clamped = math_min_s32(math_max_s32(y_scaled, smin), smax); |
| 70 | const int32_t z_clamped = math_min_s32(math_max_s32(z_scaled, smin), smax); |
| 71 | const int32_t w_clamped = math_min_s32(math_max_s32(w_scaled, smin), smax); |
| 72 | |
| 73 | // Add zero point to clamped value. |
| 74 | // The result is guaranteed to be in [qmin, qmax] range. |
| 75 | // |
| 76 | // This addition can not be safely done before clamping, because scaled values are in [-2147483520, 2147483519] |
| 77 | // range, so addition of zero point (which can be up to 127) can overflow signed 32-bit integer. |
| 78 | const int32_t x_biased = x_clamped + zero_point; |
| 79 | const int32_t y_biased = y_clamped + zero_point; |
| 80 | const int32_t z_biased = z_clamped + zero_point; |
| 81 | const int32_t w_biased = w_clamped + zero_point; |
| 82 | |
| 83 | output[0] = (int8_t) x_biased; |
| 84 | output[1] = (int8_t) y_biased; |
| 85 | output[2] = (int8_t) z_biased; |
| 86 | output[3] = (int8_t) w_biased; |
| 87 | output += 4; |
| 88 | } |
| 89 | } |