| // Copyright 2020 Google LLC |
| // |
| // This source code is licensed under the BSD-style license found in the |
| // LICENSE file in the root directory of this source tree. |
| |
| $ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" |
| $assert CHANNEL_TILE % 8 == 0 |
| $assert CHANNEL_TILE >= 8 |
| $assert KERNEL_TILE >= 2 |
| #include <assert.h> |
| |
| #include <arm_neon.h> |
| |
| #include <xnnpack/dwconv.h> |
| |
| |
| void xnn_qs8_dwconv_minmax_ukernel_up${CHANNEL_TILE}x${KERNEL_TILE}__neon_mul16( |
| size_t channels, |
| size_t output_width, |
| const int8_t** input, |
| const void* weights, |
| int8_t* output, |
| size_t input_stride, |
| size_t output_increment, |
| size_t input_offset, |
| const int8_t* zero, |
| const union xnn_qs8_gemm_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN |
| { |
| assert(channels != 0); |
| assert(output_width != 0); |
| |
| const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier); |
| const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift); |
| const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0))); |
| const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point); |
| $if CHANNEL_TILE == 8: |
| const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min); |
| const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max); |
| $else: |
| const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min); |
| const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max); |
| do { |
| $for K in range(KERNEL_TILE): |
| const int8_t* i${K} = input[${K}]; |
| assert(i${K} != NULL); |
| if XNN_UNPREDICTABLE(i${K} != zero) { |
| i${K} = (const int8_t*) ((uintptr_t) i${K} + input_offset); |
| } |
| input = (const int8_t**) ((uintptr_t) input + input_stride); |
| |
| size_t c = channels; |
| const void* w = weights; |
| for (; c >= ${CHANNEL_TILE}; c -= ${CHANNEL_TILE}) { |
| $for C in range(0, CHANNEL_TILE, 4): |
| int32x4_t vacc${ABC[C:C+4]} = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)); |
| |
| $for K in range(KERNEL_TILE): |
| |
| $for C in range(0, CHANNEL_TILE, 8): |
| const int16x8_t vi${K}x${ABC[C:C+8]} = vmovl_s8(vld1_s8(i${K})); i${K} += 8; |
| const int16x8_t vk${K}x${ABC[C:C+8]} = vmovl_s8(vld1_s8(w)); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t)); |
| |
| $for C in range(0, CHANNEL_TILE, 8): |
| vacc${ABC[C:C+4]} = vmlal_s16(vacc${ABC[C:C+4]}, vget_low_s16(vi${K}x${ABC[C:C+8]}), vget_low_s16(vk${K}x${ABC[C:C+8]})); |
| vacc${ABC[C+4:C+8]} = vmlal_s16(vacc${ABC[C+4:C+8]}, vget_high_s16(vi${K}x${ABC[C:C+8]}), vget_high_s16(vk${K}x${ABC[C:C+8]})); |
| |
| $for C in range(0, CHANNEL_TILE, 4): |
| vacc${ABC[C:C+4]} = vqrdmulhq_s32(vacc${ABC[C:C+4]}, vmultiplier); |
| |
| $for C in range(0, CHANNEL_TILE, 4): |
| vacc${ABC[C:C+4]} = vsraq_n_s32(vacc${ABC[C:C+4]}, vbicq_s32(vacc${ABC[C:C+4]}, vzero_shift_mask), 31); |
| |
| $for C in range(0, CHANNEL_TILE, 4): |
| vacc${ABC[C:C+4]} = vrshlq_s32(vacc${ABC[C:C+4]}, vright_shift); |
| |
| #if XNN_ARCH_ARM64 |
| $for C in range(0, CHANNEL_TILE, 8): |
| const int16x8_t vacc${ABC[C:C+8]} = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc${ABC[C:C+4]}), vacc${ABC[C+4:C+8]}), voutput_zero_point); |
| |
| $for C in range(0, CHANNEL_TILE, 16): |
| $if C + 8 < CHANNEL_TILE: |
| int8x16_t vout${ABC[C:C+16]} = vqmovn_high_s16(vqmovn_s16(vacc${ABC[C:C+8]}), vacc${ABC[C+8:C+16]}); |
| $else: |
| int8x8_t vout${ABC[C:C+8]} = vqmovn_s16(vacc${ABC[C:C+8]}); |
| #else |
| $for C in range(0, CHANNEL_TILE, 8): |
| const int16x8_t vacc${ABC[C:C+8]} = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc${ABC[C:C+4]}), vqmovn_s32(vacc${ABC[C+4:C+8]})), voutput_zero_point); |
| |
| $for C in range(0, CHANNEL_TILE, 16): |
| $if C + 8 < CHANNEL_TILE: |
| int8x16_t vout${ABC[C:C+16]} = vcombine_s8(vqmovn_s16(vacc${ABC[C:C+8]}), vqmovn_s16(vacc${ABC[C+8:C+16]})); |
| $else: |
| int8x8_t vout${ABC[C:C+8]} = vqmovn_s16(vacc${ABC[C:C+8]}); |
| #endif |
| |
| $for C in range(0, CHANNEL_TILE, 16): |
| $if C + 8 < CHANNEL_TILE: |
| vout${ABC[C:C+16]} = vmaxq_s8(vout${ABC[C:C+16]}, voutput_min); |
| $else: |
| $if CHANNEL_TILE == 8: |
| vout${ABC[C:C+8]} = vmax_s8(vout${ABC[C:C+8]}, voutput_min); |
| $else: |
| vout${ABC[C:C+8]} = vmax_s8(vout${ABC[C:C+8]}, vget_low_s8(voutput_min)); |
| |
| $for C in range(0, CHANNEL_TILE, 16): |
| $if C + 8 < CHANNEL_TILE: |
| vout${ABC[C:C+16]} = vminq_s8(vout${ABC[C:C+16]}, voutput_max); |
| $else: |
| $if CHANNEL_TILE == 8: |
| vout${ABC[C:C+8]} = vmin_s8(vout${ABC[C:C+8]}, voutput_max); |
| $else: |
| vout${ABC[C:C+8]} = vmin_s8(vout${ABC[C:C+8]}, vget_low_s8(voutput_max)); |
| |
| $for C in range(0, CHANNEL_TILE, 16): |
| $if C + 8 < CHANNEL_TILE: |
| vst1q_s8(output, vout${ABC[C:C+16]}); output += 16; |
| $else: |
| vst1_s8(output, vout${ABC[C:C+8]}); output += 8; |
| } |
| if XNN_UNLIKELY(c != 0) { |
| $if CHANNEL_TILE > 8: |
| const int8_t* k = (const int8_t*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(int32_t)); |
| ${"do " if CHANNEL_TILE > 8 else ""}{ |
| int32x4_t vacc${ABC[0:4]} = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)); |
| int32x4_t vacc${ABC[4:8]} = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)); |
| |
| $for K in range(KERNEL_TILE): |
| $if CHANNEL_TILE > 8: |
| const int16x8_t vi${K}x${ABC[0:8]} = vmovl_s8(vld1_s8(i${K})); i${K} += 8; |
| $else: |
| const int16x8_t vi${K}x${ABC[0:8]} = vmovl_s8(vld1_s8(i${K})); |
| $if CHANNEL_TILE > 8: |
| $if K == 0: |
| const int16x8_t vk${K}x${ABC[0:8]} = vmovl_s8(vld1_s8(k)); k += 8; |
| $else: |
| const int16x8_t vk${K}x${ABC[0:8]} = vmovl_s8(vld1_s8((const void*) (k + ${K * CHANNEL_TILE - 8}))); |
| $else: |
| $if K == 0: |
| const int16x8_t vk${K}x${ABC[0:8]} = vmovl_s8(vld1_s8(w)); |
| $else: |
| const int16x8_t vk${K}x${ABC[0:8]} = vmovl_s8(vld1_s8((const void*) ((uintptr_t) w + ${K * CHANNEL_TILE} * sizeof(int8_t)))); |
| |
| vacc${ABC[0:4]} = vmlal_s16(vacc${ABC[0:4]}, vget_low_s16(vi${K}x${ABC[0:8]}), vget_low_s16(vk${K}x${ABC[0:8]})); |
| vacc${ABC[4:8]} = vmlal_s16(vacc${ABC[4:8]}, vget_high_s16(vi${K}x${ABC[0:8]}), vget_high_s16(vk${K}x${ABC[0:8]})); |
| |
| vacc${ABC[0:4]} = vqrdmulhq_s32(vacc${ABC[0:4]}, vmultiplier); |
| vacc${ABC[4:8]} = vqrdmulhq_s32(vacc${ABC[4:8]}, vmultiplier); |
| |
| vacc${ABC[0:4]} = vsraq_n_s32(vacc${ABC[0:4]}, vbicq_s32(vacc${ABC[0:4]}, vzero_shift_mask), 31); |
| vacc${ABC[4:8]} = vsraq_n_s32(vacc${ABC[4:8]}, vbicq_s32(vacc${ABC[4:8]}, vzero_shift_mask), 31); |
| |
| vacc${ABC[0:4]} = vrshlq_s32(vacc${ABC[0:4]}, vright_shift); |
| vacc${ABC[4:8]} = vrshlq_s32(vacc${ABC[4:8]}, vright_shift); |
| |
| #if XNN_ARCH_ARM64 |
| const int16x8_t vacc${ABC[0:8]} = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc${ABC[0:4]}), vacc${ABC[4:8]}), voutput_zero_point); |
| |
| int8x8_t vout${ABC[0:8]} = vqmovn_s16(vacc${ABC[0:8]}); |
| #else |
| const int16x8_t vacc${ABC[0:8]} = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc${ABC[0:4]}), vqmovn_s32(vacc${ABC[4:8]})), voutput_zero_point); |
| |
| int8x8_t vout${ABC[0:8]} = vqmovn_s16(vacc${ABC[0:8]}); |
| #endif |
| |
| $if CHANNEL_TILE == 8: |
| vout${ABC[0:8]} = vmax_s8(vout${ABC[0:8]}, voutput_min); |
| vout${ABC[0:8]} = vmin_s8(vout${ABC[0:8]}, voutput_max); |
| $else: |
| vout${ABC[0:8]} = vmax_s8(vout${ABC[0:8]}, vget_low_s8(voutput_min)); |
| vout${ABC[0:8]} = vmin_s8(vout${ABC[0:8]}, vget_low_s8(voutput_max)); |
| |
| $if CHANNEL_TILE > 8: |
| if XNN_LIKELY(c >= 8) { |
| vst1_s8(output, vout${ABC[0:8]}); output += 8; |
| c -= 8; |
| } else { |
| if (c & 4) { |
| vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout${ABC[0:8]}), 0); output += 4; |
| vout${ABC[0:8]} = vext_s8(vout${ABC[0:8]}, vout${ABC[0:8]}, 4); |
| } |
| if (c & 2) { |
| vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout${ABC[0:8]}), 0); output += 2; |
| vout${ABC[0:8]} = vext_s8(vout${ABC[0:8]}, vout${ABC[0:8]}, 2); |
| } |
| if (c & 1) { |
| vst1_lane_s8(output, vout${ABC[0:8]}, 0); output += 1; |
| } |
| c = 0; |
| } |
| $else: |
| if (c & 4) { |
| vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout${ABC[0:8]}), 0); output += 4; |
| vout${ABC[0:8]} = vext_s8(vout${ABC[0:8]}, vout${ABC[0:8]}, 4); |
| } |
| if (c & 2) { |
| vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout${ABC[0:8]}), 0); output += 2; |
| vout${ABC[0:8]} = vext_s8(vout${ABC[0:8]}, vout${ABC[0:8]}, 2); |
| } |
| if (c & 1) { |
| vst1_lane_s8(output, vout${ABC[0:8]}, 0); output += 1; |
| } |
| }${" while (c != 0);" if CHANNEL_TILE > 8 else ""} |
| } |
| |
| output = (int8_t*) ((uintptr_t) output + output_increment); |
| } while (--output_width != 0); |
| } |