Frank Barchard | b196659 | 2020-05-12 13:47:06 -0700 | [diff] [blame] | 1 | // Copyright 2020 Google LLC |
| 2 | // |
| 3 | // This source code is licensed under the BSD-style license found in the |
| 4 | // LICENSE file in the root directory of this source tree. |
| 5 | |
| 6 | $assert BATCH_TILE % 8 == 0 |
| 7 | $assert BATCH_TILE >= 8 |
| 8 | $ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" |
| 9 | #include <assert.h> |
| 10 | |
| 11 | #include <arm_neon.h> |
| 12 | |
| 13 | #include <xnnpack/common.h> |
Marat Dukhan | a91559a | 2021-05-05 23:58:21 -0700 | [diff] [blame] | 14 | #include <xnnpack/vunary.h> |
Frank Barchard | b196659 | 2020-05-12 13:47:06 -0700 | [diff] [blame] | 15 | |
| 16 | |
Marat Dukhan | 6674d69 | 2021-05-05 22:27:00 -0700 | [diff] [blame] | 17 | void xnn_f16_vhswish_ukernel__neonfp16arith_x${BATCH_TILE}( |
Frank Barchard | b196659 | 2020-05-12 13:47:06 -0700 | [diff] [blame] | 18 | size_t n, |
| 19 | const void* restrict x_ptr, |
| 20 | void* restrict y_ptr, |
Marat Dukhan | 14dd8d0 | 2022-01-06 16:03:31 -0800 | [diff] [blame] | 21 | const union xnn_f16_hswish_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS |
Frank Barchard | b196659 | 2020-05-12 13:47:06 -0700 | [diff] [blame] | 22 | { |
| 23 | assert(n != 0); |
| 24 | assert(n % sizeof(__fp16) == 0); |
| 25 | |
Frank Barchard | febfe59 | 2020-06-03 14:10:38 -0700 | [diff] [blame] | 26 | const __fp16* x = (const __fp16*) x_ptr; |
| 27 | __fp16* y = (__fp16*) y_ptr; |
Frank Barchard | b196659 | 2020-05-12 13:47:06 -0700 | [diff] [blame] | 28 | |
Marat Dukhan | 14dd8d0 | 2022-01-06 16:03:31 -0800 | [diff] [blame] | 29 | const float16x8_t vsixth = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->neon.sixth)); |
| 30 | const float16x8_t vthree = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->neon.three)); |
| 31 | const int16x8_t vsix = vreinterpretq_s16_u16(vld1q_dup_u16(¶ms->neon.six)); |
Marat Dukhan | 201ea0e | 2020-07-10 22:47:19 -0700 | [diff] [blame] | 32 | const int16x8_t vzero = vdupq_n_s16(0); |
Frank Barchard | b196659 | 2020-05-12 13:47:06 -0700 | [diff] [blame] | 33 | |
Frank Barchard | e8466f4 | 2020-06-08 17:05:01 -0700 | [diff] [blame] | 34 | $if BATCH_TILE > 8: |
Marat Dukhan | 201ea0e | 2020-07-10 22:47:19 -0700 | [diff] [blame] | 35 | for (; n >= ${BATCH_TILE} * sizeof(__fp16); n -= ${BATCH_TILE} * sizeof(__fp16)) { |
| 36 | $for N in range(0, BATCH_TILE, 8): |
| 37 | float16x8_t vx${ABC[N:N+8]} = vld1q_f16(x); x += 8; |
| 38 | |
| 39 | $for N in range(0, BATCH_TILE, 8): |
| 40 | float16x8_t vacc${ABC[N:N+8]} = vaddq_f16(vx${ABC[N:N+8]}, vthree); |
| 41 | vx${ABC[N:N+8]} = vmulq_f16(vx${ABC[N:N+8]}, vsixth); |
| 42 | |
| 43 | $for N in range(0, BATCH_TILE, 8): |
| 44 | vacc${ABC[N:N+8]} = vreinterpretq_f16_s16(vmaxq_s16(vreinterpretq_s16_f16(vacc${ABC[N:N+8]}), vzero)); |
| 45 | |
| 46 | $for N in range(0, BATCH_TILE, 8): |
| 47 | vacc${ABC[N:N+8]} = vreinterpretq_f16_s16(vminq_s16(vreinterpretq_s16_f16(vacc${ABC[N:N+8]}), vsix)); |
| 48 | |
| 49 | $for N in range(0, BATCH_TILE, 8): |
| 50 | vacc${ABC[N:N+8]} = vmulq_f16(vacc${ABC[N:N+8]}, vx${ABC[N:N+8]}); |
| 51 | |
| 52 | $for N in range(0, BATCH_TILE, 8): |
| 53 | vst1q_f16(y, vacc${ABC[N:N+8]}); y += 8; |
Frank Barchard | b196659 | 2020-05-12 13:47:06 -0700 | [diff] [blame] | 54 | } |
Marat Dukhan | 201ea0e | 2020-07-10 22:47:19 -0700 | [diff] [blame] | 55 | for (; n >= 8 * sizeof(__fp16); n -= 8 * sizeof(__fp16)) { |
| 56 | float16x8_t vx = vld1q_f16(x); x += 8; |
| 57 | float16x8_t vacc = vaddq_f16(vx, vthree); |
| 58 | vx = vmulq_f16(vx, vsixth); |
| 59 | vacc = vreinterpretq_f16_s16(vmaxq_s16(vreinterpretq_s16_f16(vacc), vzero)); |
| 60 | vacc = vreinterpretq_f16_s16(vminq_s16(vreinterpretq_s16_f16(vacc), vsix)); |
| 61 | vacc = vmulq_f16(vacc, vx); |
| 62 | vst1q_f16(y, vacc); y += 8; |
| 63 | } |
Frank Barchard | b196659 | 2020-05-12 13:47:06 -0700 | [diff] [blame] | 64 | if XNN_UNLIKELY(n != 0) { |
Marat Dukhan | 201ea0e | 2020-07-10 22:47:19 -0700 | [diff] [blame] | 65 | float16x8_t vx = vld1q_f16(x); |
| 66 | float16x8_t vacc = vaddq_f16(vx, vthree); |
| 67 | vx = vmulq_f16(vx, vsixth); |
| 68 | vacc = vreinterpretq_f16_s16(vmaxq_s16(vreinterpretq_s16_f16(vacc), vzero)); |
| 69 | vacc = vreinterpretq_f16_s16(vminq_s16(vreinterpretq_s16_f16(vacc), vsix)); |
| 70 | vacc = vmulq_f16(vacc, vx); |
Frank Barchard | b196659 | 2020-05-12 13:47:06 -0700 | [diff] [blame] | 71 | |
Marat Dukhan | 201ea0e | 2020-07-10 22:47:19 -0700 | [diff] [blame] | 72 | float16x4_t vacc_lo = vget_low_f16(vacc); |
Frank Barchard | b196659 | 2020-05-12 13:47:06 -0700 | [diff] [blame] | 73 | if (n & (4 * sizeof(__fp16))) { |
Marat Dukhan | 201ea0e | 2020-07-10 22:47:19 -0700 | [diff] [blame] | 74 | vst1_f16(y, vacc_lo); y += 4; |
| 75 | vacc_lo = vget_high_f16(vacc); |
Frank Barchard | b196659 | 2020-05-12 13:47:06 -0700 | [diff] [blame] | 76 | } |
Frank Barchard | b196659 | 2020-05-12 13:47:06 -0700 | [diff] [blame] | 77 | if (n & (2 * sizeof(__fp16))) { |
Marat Dukhan | 5f7cf55 | 2021-11-25 17:37:03 -0800 | [diff] [blame] | 78 | vst1_lane_u32((void*) y, vreinterpret_u32_f16(vacc_lo), 0); y += 2; |
Marat Dukhan | 201ea0e | 2020-07-10 22:47:19 -0700 | [diff] [blame] | 79 | vacc_lo = vext_f16(vacc_lo, vacc_lo, 2); |
Frank Barchard | b196659 | 2020-05-12 13:47:06 -0700 | [diff] [blame] | 80 | } |
Frank Barchard | b196659 | 2020-05-12 13:47:06 -0700 | [diff] [blame] | 81 | if (n & (1 * sizeof(__fp16))) { |
Marat Dukhan | 201ea0e | 2020-07-10 22:47:19 -0700 | [diff] [blame] | 82 | vst1_lane_f16(y, vacc_lo, 0); |
Frank Barchard | b196659 | 2020-05-12 13:47:06 -0700 | [diff] [blame] | 83 | } |
| 84 | } |
| 85 | } |