Marat Dukhan | c9852ba | 2020-05-13 17:21:29 -0700 | [diff] [blame] | 1 | // Copyright 2020 Google LLC |
| 2 | // |
| 3 | // This source code is licensed under the BSD-style license found in the |
| 4 | // LICENSE file in the root directory of this source tree. |
| 5 | |
| 6 | #include <assert.h> |
| 7 | #include <stddef.h> |
| 8 | #include <stdint.h> |
| 9 | |
| 10 | #include <arm_neon.h> |
| 11 | |
| 12 | #include <xnnpack/math-stubs.h> |
| 13 | |
| 14 | |
| 15 | void xnn_math_f32_roundu__neon_addsub( |
| 16 | size_t n, |
| 17 | const float* input, |
| 18 | float* output) |
| 19 | { |
| 20 | assert(n % (4 * sizeof(float)) == 0); |
| 21 | |
| 22 | // Addition of this number to a floating-point number x cause rounding of the result to an integer. Then this magic |
| 23 | // number is subtracted back from the result to get original x rounded to integer. This trick works only for |
| 24 | // 0 <= x < 2**24, but all numbers in 2**23 <= x < 2**24 range are integers, so we can further restrict it to |
| 25 | // 0 <= x < 2**23. Then the upper bound of the validity interval is conveniently the same as the magic number. |
| 26 | const float32x4_t vmagic_number = vmovq_n_f32(0x1.000000p+23f); |
| 27 | // Mask for the sign bit of a floating-point number. |
| 28 | const uint32x4_t vsign_mask = vmovq_n_u32(UINT32_C(0x80000000)); |
| 29 | // Unit constant to increment results rounded "wrong way" (i.e. down) in the round-to-nearest-even operation. |
| 30 | const float32x4_t vone = vmovq_n_f32(1.0f); |
| 31 | |
| 32 | for (; n != 0; n -= 4 * sizeof(float)) { |
| 33 | const float32x4_t vx = vld1q_f32(input); input += 4; |
| 34 | |
| 35 | // The rounding trick works only for x >= 0, so we compute absolute value of x, round it, and restore the sign in |
| 36 | // the end. This method works for round-to-nearest-even because it is an odd function. |
| 37 | const float32x4_t vabsx = vabsq_f32(vx); |
| 38 | // Compute bitmask for the bits we want to copy from the rounded abs(x). Other bits will be copied from x. |
| 39 | // If abs(x) >= 2**23, we want all bits from x. |
| 40 | // If abs(x) < 2**23 or x is NaN, we want all but the sign bit from the rounded abs(x) and the sign bit from x. |
| 41 | // Note: we do vcaltq_f32(vmagic_number, vx) instead of vcltq_f32(vmagic_number, vabsx) to reduce dependency chain. |
| 42 | const uint32x4_t vrndmask = vorrq_u32(vcaltq_f32(vmagic_number, vx), vsign_mask); |
| 43 | |
| 44 | // Addition-subtraction trick with the magic number to cause rounding to the nearest-even integer for abs(x). |
| 45 | // Note: the result is valid only for 0 <= abs(x) < 2**23. |
| 46 | // Note: addition-subtraction implicitly converts SNaN inputs to QNaNs. |
| 47 | const float32x4_t vrndabsx = vsubq_f32(vaddq_f32(vabsx, vmagic_number), vmagic_number); |
| 48 | |
| 49 | // Combine abs(x) rounded via addition-subtraction trick and the input x value. |
| 50 | // For abs(x) < 2**23, the result is abs(x) rounded via addition-subtraction trick with the sign of x. |
| 51 | // For NaN inputs, the result is x converted to QNaN as a side-effect of addition-subtraction. |
| 52 | // For abs(x) >= 2**23, the result is x itself. |
| 53 | const float32x4_t vrndx = vbslq_f32(vrndmask, vx, vrndabsx); |
| 54 | |
| 55 | // Compute bitmask for the bits to copy from the adjusted rounded x. Other bits will be copied from rounded x. |
| 56 | // If rounded x < x, we want all but the sign bit from the adjusted rounded x and the sign bit from rounded x (same |
| 57 | // as the sign bit of x). |
| 58 | // If rounded x >= x or rounded x is NaN (implies x is NaN), we want all bits from rounded x. |
| 59 | const uint32x4_t vadjmask = vbicq_u32(vcltq_f32(vrndx, vx), vsign_mask); |
| 60 | // Adjust the rounded x value. |
| 61 | // The adjusted value is a unit above the rounded-to-nearest-even x value, but is used only if the rounded value is |
| 62 | // below x. In these cases, the adjusted value is x rounded up. |
| 63 | const float32x4_t vadjrndx = vaddq_f32(vrndx, vone); |
| 64 | |
| 65 | // Combine the adjusted rounded x and the original rounded to nearest-even x. |
| 66 | // For rounded x < x, the result is the absolute value of adjusted rounded-to-nearest-even x with the sign of |
| 67 | // rounded-to-nearest-even x (same as sign of x). Propagating the sign of x is important to produce negative zero |
| 68 | // for -1.0 < x < -0.5 inputs, where otherwise we would get -1.0 (rounded x) + 1.0 (adjustment) = +0.0. |
| 69 | // For rounded x >= x, the result is the rounded-to-nearest-even x. |
| 70 | // For NaN inputs, the result is rounded x (same as x converted to QNaN as a side-effect of addition-subtraction). |
| 71 | const float32x4_t vy = vbslq_f32(vadjmask, vadjrndx, vrndx); |
| 72 | |
| 73 | vst1q_f32(output, vy); output += 4; |
| 74 | } |
| 75 | } |