Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / XNNPACK / 66ae257005f9963d3d5ceafe91640474dcfe19c3 / . / src / qs8-igemm / neon-mlal-lane.c.in
blob: b6d53ed24d92307f06bfd18f1b18d1902d108b19 [file] [log] [blame]
// Copyright 2020 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
$assert NR % 8 == 0
$assert 8 <= NR <= 16
$assert REQUANTIZATION in ["FP32", "GEMMLOWP", "RNDNU"]
$assert DATATYPE in ["QC8", "QS8", "QU8"]
$assert DATATYPE != "QC8" or REQUANTIZATION == "FP32"
#include <assert.h>
#include <arm_neon.h>
#include <xnnpack/common.h>
#include <xnnpack/igemm.h>
$if REQUANTIZATION == "FP32" and ARMV8:
#include <xnnpack/intrinsics-polyfill.h>
$PARAMS_UNION = "xnn_qs8_minmax_params" if DATATYPE == "QC8" else "xnn_%s_conv_minmax_params" % DATATYPE.lower()
$PARAMS_STRUCT = ("" if DATATYPE == "QC8" else REQUANTIZATION.lower() + "_") + ("neonv8" if ARMV8 and DATATYPE != "QC8" else "neon")
$if REQUANTIZATION == "FP32" and DATATYPE == "QC8" and not ARMV8:
$PARAMS_STRUCT = "neon_fp32"
$XINT8_T = "uint8_t" if DATATYPE == "QU8" else "int8_t"
$XINT8X8_T = "uint8x8_t" if DATATYPE == "QU8" else "int8x8_t"
$XINT8X16_T = "uint8x16_t" if DATATYPE == "QU8" else "int8x16_t"
$VGET_LOW_X8 = "vget_low_u8" if DATATYPE == "QU8" else "vget_low_s8"
$VGET_HIGH_X8 = "vget_high_u8" if DATATYPE == "QU8" else "vget_high_s8"
$VCOMBINE_X8 = "vcombine_u8" if DATATYPE == "QU8" else "vcombine_s8"
$VREINTERPRET_U32_X8 = "vreinterpret_u32_u8" if DATATYPE == "QU8" else "vreinterpret_u32_s8"
$VREINTERPRETQ_U32_X8 = "vreinterpretq_u32_u8" if DATATYPE == "QU8" else "vreinterpretq_u32_s8"
$VREINTERPRET_U16_X8 = "vreinterpret_u16_u8" if DATATYPE == "QU8" else "vreinterpret_u16_s8"
$VREINTERPRETQ_U16_X8 = "vreinterpretq_u16_u8" if DATATYPE == "QU8" else "vreinterpretq_u16_s8"
$VREINTERPRETQ_X8_S16 = "vreinterpretq_u8_s16" if DATATYPE == "QU8" else "vreinterpretq_s8_s16"
$VLD1_X8 = "vld1_u8" if DATATYPE == "QU8" else "vld1_s8"
$VLD1_DUP_X8 = "vld1_dup_u8" if DATATYPE == "QU8" else "vld1_dup_s8"
$VLD1Q_DUP_X8 = "vld1q_dup_u8" if DATATYPE == "QU8" else "vld1q_dup_s8"
$VST1_X8 = "vst1_u8" if DATATYPE == "QU8" else "vst1_s8"
$VST1Q_X8 = "vst1q_u8" if DATATYPE == "QU8" else "vst1q_s8"
$VST1_LANE_X8 = "vst1_lane_u8" if DATATYPE == "QU8" else "vst1_lane_s8"
$VST1Q_LANE_X8 = "vst1q_lane_u8" if DATATYPE == "QU8" else "vst1q_lane_s8"
$VMIN_X8 = "vmin_u8" if DATATYPE == "QU8" else "vmin_s8"
$VMAX_X8 = "vmax_u8" if DATATYPE == "QU8" else "vmax_s8"
$VMINQ_X8 = "vminq_u8" if DATATYPE == "QU8" else "vminq_s8"
$VMAXQ_X8 = "vmaxq_u8" if DATATYPE == "QU8" else "vmaxq_s8"
$VEXT_X8 = "vext_u8" if DATATYPE == "QU8" else "vext_s8"
$VEXTQ_X8 = "vextq_u8" if DATATYPE == "QU8" else "vextq_s8"
$VQMOVXN_S16 = "vqmovun_s16" if DATATYPE == "QU8" else "vqmovn_s16"
$VQMOVXN_HIGH_S16 = "vqmovun_high_s16" if DATATYPE == "QU8" else "vqmovn_high_s16"
$VMOVN_X16 = "vmovn_u16" if DATATYPE == "QU8" else "vmovn_s16"
$VUZP1Q_X8 = "vuzp1q_u8" if DATATYPE == "QU8" else "vuzp1q_s8"
$ISA = "neonv8" if ARMV8 else "neon"
void xnn_${DATATYPE.lower()}_igemm_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x${NR}__${ISA}_mlal_lane${"_prfm" if PREFETCH else ""}(
size_t mr,
size_t nc,
size_t kc,
size_t ks,
const ${XINT8_T}** restrict a,
const void* restrict w,
${XINT8_T}* restrict c,
size_t cm_stride,
size_t cn_stride,
size_t a_offset,
const ${XINT8_T}* zero,
const union ${PARAMS_UNION} params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
{
assert(mr != 0);
assert(mr <= ${MR});
assert(nc != 0);
assert(kc != 0);
assert(ks != 0);
assert(ks % (${MR} * sizeof(void*)) == 0);
assert(a_offset % sizeof(${XINT8_T}) == 0);
assert(a != NULL);
assert(w != NULL);
assert(c != NULL);
${XINT8_T}* c0 = c;
$for M in range(1, MR):
${XINT8_T}* c${M} = (${XINT8_T}*) ((uintptr_t) c${M-1} + cm_stride);
$if M % 2 == 0:
if XNN_UNPREDICTABLE(mr <= ${M}) {
c${M} = c${M-1};
}
$elif M + 1 == MR:
if XNN_UNPREDICTABLE(mr != ${M+1}) {
c${M} = c${M-1};
}
$else:
if XNN_UNPREDICTABLE(mr < ${M+1}) {
c${M} = c${M-1};
}
$if DATATYPE == "QU8":
const uint8x8_t vb_zero_point = vld1_dup_u8(&params->${PARAMS_STRUCT}.kernel_zero_point[0]);
do {
$for N in range(0, NR, 4):
int32x4_t vacc0x${ABC[N:N+4]} = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
$for M in range(1, MR):
$for N in range(0, NR, 4):
int32x4_t vacc${M}x${ABC[N:N+4]} = vacc0x${ABC[N:N+4]};
size_t p = ks;
do {
$for M in range(MR):
const ${XINT8_T}* restrict a${M} = a[${M}];
if XNN_UNPREDICTABLE(a${M} != zero) {
a${M} = (const ${XINT8_T}*) ((uintptr_t) a${M} + a_offset);
}
a += ${MR};
size_t k = kc;
while (k >= 8 * sizeof(${XINT8_T})) {
$for M in range(MR):
const ${XINT8X8_T} va${M} = ${VLD1_X8}(a${M}); a${M} += 8;
$if DATATYPE == "QU8":
const int16x8_t vxa${M} = vreinterpretq_s16_u16(vmovl_u8(va${M}));
$else:
const int16x8_t vxa${M} = vmovl_s8(va${M});
$for K in range(4):
$for N in range(0, NR, 8):
const ${XINT8X8_T} vb${ABC[N:N+8]}c${K} = ${VLD1_X8}(w); w = (const void*) ((const ${XINT8_T}*) w + 8);
$if DATATYPE == "QU8":
const int16x8_t vxb${ABC[N:N+8]}c${K} = vreinterpretq_s16_u16(vsubl_u8(vb${ABC[N:N+8]}c${K}, vb_zero_point));
$else:
const int16x8_t vxb${ABC[N:N+8]}c${K} = vmovl_s8(vb${ABC[N:N+8]}c${K});
$for M in range(MR):
vacc${M}x${ABC[N:N+4]} = vmlal_lane_s16(vacc${M}x${ABC[N:N+4]}, vget_low_s16(vxb${ABC[N:N+8]}c${K}), vget_low_s16(vxa${M}), ${K});
vacc${M}x${ABC[N+4:N+8]} = vmlal_lane_s16(vacc${M}x${ABC[N+4:N+8]}, vget_high_s16(vxb${ABC[N:N+8]}c${K}), vget_low_s16(vxa${M}), ${K});
$if PREFETCH:
$for N in range(0, NR, 8):
__builtin_prefetch((const ${XINT8_T}*) w + ${N * 8 + 480});
$for K in range(4, 8):
$for N in range(0, NR, 8):
const ${XINT8X8_T} vb${ABC[N:N+8]}c${K} = ${VLD1_X8}(w); w = (const void*) ((const ${XINT8_T}*) w + 8);
$if DATATYPE == "QU8":
const int16x8_t vxb${ABC[N:N+8]}c${K} = vreinterpretq_s16_u16(vsubl_u8(vb${ABC[N:N+8]}c${K}, vb_zero_point));
$else:
const int16x8_t vxb${ABC[N:N+8]}c${K} = vmovl_s8(vb${ABC[N:N+8]}c${K});
$for M in range(MR):
vacc${M}x${ABC[N:N+4]} = vmlal_lane_s16(vacc${M}x${ABC[N:N+4]}, vget_low_s16(vxb${ABC[N:N+8]}c${K}), vget_high_s16(vxa${M}), ${K-4});
vacc${M}x${ABC[N+4:N+8]} = vmlal_lane_s16(vacc${M}x${ABC[N+4:N+8]}, vget_high_s16(vxb${ABC[N:N+8]}c${K}), vget_high_s16(vxa${M}), ${K-4});
k -= 8 * sizeof(${XINT8_T});
}
if XNN_UNLIKELY(k != 0) {
$for M in range(MR):
const ${XINT8X8_T} va${M} = ${VLD1_X8}(a${M}); a${M} = (const ${XINT8_T}*) ((uintptr_t) a${M} + k);
$if DATATYPE == "QU8":
const int16x8_t vxa${M} = vreinterpretq_s16_u16(vmovl_u8(va${M}));
$else:
const int16x8_t vxa${M} = vmovl_s8(va${M});
$for N in range(0, NR, 8):
const ${XINT8X8_T} vb${ABC[N:N+8]}c0 = ${VLD1_X8}(w); w = (const void*) ((const ${XINT8_T}*) w + 8);
$if DATATYPE == "QU8":
const int16x8_t vxb${ABC[N:N+8]}c0 = vreinterpretq_s16_u16(vsubl_u8(vb${ABC[N:N+8]}c0, vb_zero_point));
$else:
const int16x8_t vxb${ABC[N:N+8]}c0 = vmovl_s8(vb${ABC[N:N+8]}c0);
$for M in range(MR):
$for N in range(0, NR, 8):
vacc${M}x${ABC[N:N+4]} = vmlal_lane_s16(vacc${M}x${ABC[N:N+4]}, vget_low_s16(vxb${ABC[N:N+8]}c0), vget_low_s16(vxa${M}), 0);
vacc${M}x${ABC[N+4:N+8]} = vmlal_lane_s16(vacc${M}x${ABC[N+4:N+8]}, vget_high_s16(vxb${ABC[N:N+8]}c0), vget_low_s16(vxa${M}), 0);
if (k >= 2 * sizeof(${XINT8_T})) {
$for N in range(0, NR, 8):
const ${XINT8X8_T} vb${ABC[N:N+8]}c1 = ${VLD1_X8}(w); w = (const void*) ((const ${XINT8_T}*) w + 8);
$if DATATYPE == "QU8":
const int16x8_t vxb${ABC[N:N+8]}c1 = vreinterpretq_s16_u16(vsubl_u8(vb${ABC[N:N+8]}c1, vb_zero_point));
$else:
const int16x8_t vxb${ABC[N:N+8]}c1 = vmovl_s8(vb${ABC[N:N+8]}c1);
$for M in range(MR):
$for N in range(0, NR, 8):
vacc${M}x${ABC[N:N+4]} = vmlal_lane_s16(vacc${M}x${ABC[N:N+4]}, vget_low_s16(vxb${ABC[N:N+8]}c1), vget_low_s16(vxa${M}), 1);
vacc${M}x${ABC[N+4:N+8]} = vmlal_lane_s16(vacc${M}x${ABC[N+4:N+8]}, vget_high_s16(vxb${ABC[N:N+8]}c1), vget_low_s16(vxa${M}), 1);
if (k > 2 * sizeof(${XINT8_T})) {
$for N in range(0, NR, 8):
const ${XINT8X8_T} vb${ABC[N:N+8]}c2 = ${VLD1_X8}(w); w = (const void*) ((const ${XINT8_T}*) w + 8);
$if DATATYPE == "QU8":
const int16x8_t vxb${ABC[N:N+8]}c2 = vreinterpretq_s16_u16(vsubl_u8(vb${ABC[N:N+8]}c2, vb_zero_point));
$else:
const int16x8_t vxb${ABC[N:N+8]}c2 = vmovl_s8(vb${ABC[N:N+8]}c2);
$for M in range(MR):
$for N in range(0, NR, 8):
vacc${M}x${ABC[N:N+4]} = vmlal_lane_s16(vacc${M}x${ABC[N:N+4]}, vget_low_s16(vxb${ABC[N:N+8]}c2), vget_low_s16(vxa${M}), 2);
vacc${M}x${ABC[N+4:N+8]} = vmlal_lane_s16(vacc${M}x${ABC[N+4:N+8]}, vget_high_s16(vxb${ABC[N:N+8]}c2), vget_low_s16(vxa${M}), 2);
if (k >= 4 * sizeof(${XINT8_T})) {
$for N in range(0, NR, 8):
const ${XINT8X8_T} vb${ABC[N:N+8]}c3 = ${VLD1_X8}(w); w = (const void*) ((const ${XINT8_T}*) w + 8);
$if DATATYPE == "QU8":
const int16x8_t vxb${ABC[N:N+8]}c3 = vreinterpretq_s16_u16(vsubl_u8(vb${ABC[N:N+8]}c3, vb_zero_point));
$else:
const int16x8_t vxb${ABC[N:N+8]}c3 = vmovl_s8(vb${ABC[N:N+8]}c3);
$for M in range(MR):
$for N in range(0, NR, 8):
vacc${M}x${ABC[N:N+4]} = vmlal_lane_s16(vacc${M}x${ABC[N:N+4]}, vget_low_s16(vxb${ABC[N:N+8]}c3), vget_low_s16(vxa${M}), 3);
vacc${M}x${ABC[N+4:N+8]} = vmlal_lane_s16(vacc${M}x${ABC[N+4:N+8]}, vget_high_s16(vxb${ABC[N:N+8]}c3), vget_low_s16(vxa${M}), 3);
if (k > 4 * sizeof(${XINT8_T})) {
$for N in range(0, NR, 8):
const ${XINT8X8_T} vb${ABC[N:N+8]}c4 = ${VLD1_X8}(w); w = (const void*) ((const ${XINT8_T}*) w + 8);
$if DATATYPE == "QU8":
const int16x8_t vxb${ABC[N:N+8]}c4 = vreinterpretq_s16_u16(vsubl_u8(vb${ABC[N:N+8]}c4, vb_zero_point));
$else:
const int16x8_t vxb${ABC[N:N+8]}c4 = vmovl_s8(vb${ABC[N:N+8]}c4);
$for M in range(MR):
$for N in range(0, NR, 8):
vacc${M}x${ABC[N:N+4]} = vmlal_lane_s16(vacc${M}x${ABC[N:N+4]}, vget_low_s16(vxb${ABC[N:N+8]}c4), vget_high_s16(vxa${M}), 0);
vacc${M}x${ABC[N+4:N+8]} = vmlal_lane_s16(vacc${M}x${ABC[N+4:N+8]}, vget_high_s16(vxb${ABC[N:N+8]}c4), vget_high_s16(vxa${M}), 0);
if (k >= 6 * sizeof(${XINT8_T})) {
$for N in range(0, NR, 8):
const ${XINT8X8_T} vb${ABC[N:N+8]}c5 = ${VLD1_X8}(w); w = (const void*) ((const ${XINT8_T}*) w + 8);
$if DATATYPE == "QU8":
const int16x8_t vxb${ABC[N:N+8]}c5 = vreinterpretq_s16_u16(vsubl_u8(vb${ABC[N:N+8]}c5, vb_zero_point));
$else:
const int16x8_t vxb${ABC[N:N+8]}c5 = vmovl_s8(vb${ABC[N:N+8]}c5);
$for M in range(MR):
$for N in range(0, NR, 8):
vacc${M}x${ABC[N:N+4]} = vmlal_lane_s16(vacc${M}x${ABC[N:N+4]}, vget_low_s16(vxb${ABC[N:N+8]}c5), vget_high_s16(vxa${M}), 1);
vacc${M}x${ABC[N+4:N+8]} = vmlal_lane_s16(vacc${M}x${ABC[N+4:N+8]}, vget_high_s16(vxb${ABC[N:N+8]}c5), vget_high_s16(vxa${M}), 1);
if (k > 6 * sizeof(${XINT8_T})) {
$for N in range(0, NR, 8):
const ${XINT8X8_T} vb${ABC[N:N+8]}c6 = ${VLD1_X8}(w); w = (const void*) ((const ${XINT8_T}*) w + 8);
$if DATATYPE == "QU8":
const int16x8_t vxb${ABC[N:N+8]}c6 = vreinterpretq_s16_u16(vsubl_u8(vb${ABC[N:N+8]}c6, vb_zero_point));
$else:
const int16x8_t vxb${ABC[N:N+8]}c6 = vmovl_s8(vb${ABC[N:N+8]}c6);
$for M in range(MR):
$for N in range(0, NR, 8):
vacc${M}x${ABC[N:N+4]} = vmlal_lane_s16(vacc${M}x${ABC[N:N+4]}, vget_low_s16(vxb${ABC[N:N+8]}c6), vget_high_s16(vxa${M}), 2);
vacc${M}x${ABC[N+4:N+8]} = vmlal_lane_s16(vacc${M}x${ABC[N+4:N+8]}, vget_high_s16(vxb${ABC[N:N+8]}c6), vget_high_s16(vxa${M}), 2);
}
}
}
}
}
}
}
p -= ${MR} * sizeof(void*);
} while (p != 0);
// Post-accumulation work
$if REQUANTIZATION == "GEMMLOWP":
const int32x4_t vmultiplier = vld1q_dup_s32(&params->${PARAMS_STRUCT}.multiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqrdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
const int32x4_t vright_shift = vld1q_dup_s32(&params->${PARAMS_STRUCT}.right_shift);
const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0)));
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vsraq_n_s32(vacc${M}x${ABC[N:N+4]}, vbicq_s32(vacc${M}x${ABC[N:N+4]}, vzero_shift_mask), 31);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_shift);
$elif REQUANTIZATION == "RNDNU":
const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->${PARAMS_STRUCT}.right_pre_shift);
const int32x4_t vmultiplier = vld1q_dup_s32(&params->${PARAMS_STRUCT}.multiplier);
const int32x4_t vright_post_shift = vld1q_dup_s32(&params->${PARAMS_STRUCT}.right_post_shift);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_pre_shift);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vqdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_post_shift);
$elif REQUANTIZATION == "FP32":
$for M in range(MR):
$for N in range(0, NR, 4):
float32x4_t vfpacc${M}x${ABC[N:N+4]} = vcvtq_f32_s32(vacc${M}x${ABC[N:N+4]});
$if DATATYPE == "QC8":
$for N in range(0, NR, 4):
const float32x4_t vscale${ABC[N:N+4]} = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
$for M in range(MR):
vfpacc${M}x${ABC[N:N+4]} = vmulq_f32(vfpacc${M}x${ABC[N:N+4]}, vscale${ABC[N:N+4]});
$else:
const float32x4_t vscale = vld1q_dup_f32(&params->${PARAMS_STRUCT}.scale);
$for M in range(MR):
$for N in range(0, NR, 4):
vfpacc${M}x${ABC[N:N+4]} = vmulq_f32(vfpacc${M}x${ABC[N:N+4]}, vscale);
$if ARMV8:
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vcvtnq_s32_f32(vfpacc${M}x${ABC[N:N+4]});
$else:
const float32x4_t voutput_min_less_zero_point = vld1q_dup_f32(&params->${PARAMS_STRUCT}.output_min_less_zero_point);
$for M in range(MR):
$for N in range(0, NR, 4):
vfpacc${M}x${ABC[N:N+4]} = vmaxq_f32(vfpacc${M}x${ABC[N:N+4]}, voutput_min_less_zero_point);
const float32x4_t voutput_max_less_zero_point = vld1q_dup_f32(&params->${PARAMS_STRUCT}.output_max_less_zero_point);
$for M in range(MR):
$for N in range(0, NR, 4):
vfpacc${M}x${ABC[N:N+4]} = vminq_f32(vfpacc${M}x${ABC[N:N+4]}, voutput_max_less_zero_point);
const float32x4_t vmagic_bias = vld1q_dup_f32(&params->${PARAMS_STRUCT}.magic_bias);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vreinterpretq_s32_f32(vaddq_f32(vfpacc${M}x${ABC[N:N+4]}, vmagic_bias));
const int32x4_t vmagic_bias_less_zero_point = vld1q_dup_s32(&params->${PARAMS_STRUCT}.magic_bias_less_zero_point);
$for M in range(MR):
$for N in range(0, NR, 4):
vacc${M}x${ABC[N:N+4]} = vsubq_s32(vacc${M}x${ABC[N:N+4]}, vmagic_bias_less_zero_point);
$if REQUANTIZATION != "FP32" or ARMV8:
const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->${PARAMS_STRUCT}.output_zero_point);
#if XNN_ARCH_ARM64
$if REQUANTIZATION == "FP32" and not ARMV8:
$for M in range(MR):
$for N in range(0, NR, 8):
const int16x8_t vacc${M}x${ABC[N:N+8]} = vuzp1q_s16(vreinterpretq_s16_s32(vacc${M}x${ABC[N:N+4]}), vreinterpretq_s16_s32(vacc${M}x${ABC[N+4:N+8]}));
$for M in range(MR):
$for N in range(0, NR, 16):
$if N + 8 < NR:
${XINT8X16_T} vout${M}x${ABC[N:N+16]} = ${VUZP1Q_X8}(${VREINTERPRETQ_X8_S16}(vacc${M}x${ABC[N:N+8]}), ${VREINTERPRETQ_X8_S16}(vacc${M}x${ABC[N+8:N+16]}));
$elif M % 2 == 1:
${XINT8X16_T} vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = ${VUZP1Q_X8}(${VREINTERPRETQ_X8_S16}(vacc${M-1}x${ABC[N:N+8]}), ${VREINTERPRETQ_X8_S16}(vacc${M}x${ABC[N:N+8]}));
$elif M + 1 == MR:
${XINT8X8_T} vout${M}x${ABC[N:N+8]} = ${VMOVN_X16}(vacc${M}x${ABC[N:N+8]});
$else:
$for M in range(MR):
$for N in range(0, NR, 8):
const int16x8_t vacc${M}x${ABC[N:N+8]} = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc${M}x${ABC[N:N+4]}), vacc${M}x${ABC[N+4:N+8]}), voutput_zero_point);
$for M in range(MR):
$for N in range(0, NR, 16):
$if N + 8 < NR:
${XINT8X16_T} vout${M}x${ABC[N:N+16]} = ${VQMOVXN_HIGH_S16}(${VQMOVXN_S16}(vacc${M}x${ABC[N:N+8]}), vacc${M}x${ABC[N+8:N+16]});
$elif M % 2 == 1:
${XINT8X16_T} vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = ${VQMOVXN_HIGH_S16}(${VQMOVXN_S16}(vacc${M-1}x${ABC[N:N+8]}), vacc${M}x${ABC[N:N+8]});
$elif M + 1 == MR:
${XINT8X8_T} vout${M}x${ABC[N:N+8]} = ${VQMOVXN_S16}(vacc${M}x${ABC[N:N+8]});
#else
$if REQUANTIZATION == "FP32" and not ARMV8:
$for M in range(MR):
$for N in range(0, NR, 8):
const int16x8_t vacc${M}x${ABC[N:N+8]} = vcombine_s16(vmovn_s32(vacc${M}x${ABC[N:N+4]}), vmovn_s32(vacc${M}x${ABC[N+4:N+8]}));
$if DATATYPE == "QU8":
$for M in range(MR):
$for N in range(0, NR, 16):
$if N + 8 < NR:
uint8x16_t vout${M}x${ABC[N:N+16]} = vreinterpretq_u8_s8(vcombine_s8(vmovn_s16(vacc${M}x${ABC[N:N+8]}), vmovn_s16(vacc${M}x${ABC[N+8:N+16]})));
$elif M % 2 == 1:
uint8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vreinterpretq_u8_s8(vcombine_s8(vmovn_s16(vacc${M-1}x${ABC[N:N+8]}), vmovn_s16(vacc${M}x${ABC[N:N+8]})));
$elif M + 1 == MR:
uint8x8_t vout${M}x${ABC[N:N+8]} = vreinterpret_u8_s8(vmovn_s16(vacc${M}x${ABC[N:N+8]}));
$else:
$for M in range(MR):
$for N in range(0, NR, 16):
$if N + 8 < NR:
int8x16_t vout${M}x${ABC[N:N+16]} = vcombine_s8(vmovn_s16(vacc${M}x${ABC[N:N+8]}), vmovn_s16(vacc${M}x${ABC[N+8:N+16]}));
$elif M % 2 == 1:
int8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vcombine_s8(vmovn_s16(vacc${M-1}x${ABC[N:N+8]}), vmovn_s16(vacc${M}x${ABC[N:N+8]}));
$elif M + 1 == MR:
int8x8_t vout${M}x${ABC[N:N+8]} = vmovn_s16(vacc${M}x${ABC[N:N+8]});
$else:
$for M in range(MR):
$for N in range(0, NR, 8):
const int16x8_t vacc${M}x${ABC[N:N+8]} = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc${M}x${ABC[N:N+4]}), vqmovn_s32(vacc${M}x${ABC[N+4:N+8]})), voutput_zero_point);
$for M in range(MR):
$for N in range(0, NR, 16):
$if N + 8 < NR:
${XINT8X16_T} vout${M}x${ABC[N:N+16]} = ${VCOMBINE_X8}(${VQMOVXN_S16}(vacc${M}x${ABC[N:N+8]}), ${VQMOVXN_S16}(vacc${M}x${ABC[N+8:N+16]}));
$elif M % 2 == 1:
${XINT8X16_T} vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = ${VCOMBINE_X8}(${VQMOVXN_S16}(vacc${M-1}x${ABC[N:N+8]}), ${VQMOVXN_S16}(vacc${M}x${ABC[N:N+8]}));
$elif M + 1 == MR:
${XINT8X8_T} vout${M}x${ABC[N:N+8]} = ${VQMOVXN_S16}(vacc${M}x${ABC[N:N+8]});
#endif
$if REQUANTIZATION != "FP32" or ARMV8:
$if NR == 8 and MR == 1:
const ${XINT8X8_T} voutput_min = ${VLD1_DUP_X8}(&params->${PARAMS_STRUCT}.output_min);
const ${XINT8X8_T} voutput_max = ${VLD1_DUP_X8}(&params->${PARAMS_STRUCT}.output_max);
$else:
const ${XINT8X16_T} voutput_min = ${VLD1Q_DUP_X8}(&params->${PARAMS_STRUCT}.output_min);
const ${XINT8X16_T} voutput_max = ${VLD1Q_DUP_X8}(&params->${PARAMS_STRUCT}.output_max);
$for M in range(MR):
$for N in range(0, NR, 16):
$if N + 8 < NR:
vout${M}x${ABC[N:N+16]} = ${VMAXQ_X8}(vout${M}x${ABC[N:N+16]}, voutput_min);
$elif M % 2 == 1:
vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = ${VMAXQ_X8}(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]}, voutput_min);
$elif M + 1 == MR:
$if NR == 8 and MR == 1:
vout${M}x${ABC[N:N+8]} = ${VMAX_X8}(vout${M}x${ABC[N:N+8]}, voutput_min);
$else:
vout${M}x${ABC[N:N+8]} = ${VMAX_X8}(vout${M}x${ABC[N:N+8]}, ${VGET_LOW_X8}(voutput_min));
$for M in range(MR):
$for N in range(0, NR, 16):
$if N + 8 < NR:
vout${M}x${ABC[N:N+16]} = ${VMINQ_X8}(vout${M}x${ABC[N:N+16]}, voutput_max);
$elif M % 2 == 1:
vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = ${VMINQ_X8}(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]}, voutput_max);
$elif M + 1 == MR:
$if NR == 8 and MR == 1:
vout${M}x${ABC[N:N+8]} = ${VMIN_X8}(vout${M}x${ABC[N:N+8]}, voutput_max);
$else:
vout${M}x${ABC[N:N+8]} = ${VMIN_X8}(vout${M}x${ABC[N:N+8]}, ${VGET_LOW_X8}(voutput_max));
if (nc >= ${NR}) {
$for M in reversed(range(MR)):
$for N in range(0, NR, 16):
$if N + 8 < NR:
${VST1Q_X8}(c${M} + ${N}, vout${M}x${ABC[N:N+16]});
$elif M % 2 == 1:
${VST1_X8}(c${M} + ${N}, ${VGET_HIGH_X8}(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]}));
${VST1_X8}(c${M-1} + ${N}, ${VGET_LOW_X8}(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]}));
$elif M + 1 == MR:
${VST1_X8}(c${M} + ${N}, vout${M}x${ABC[N:N+8]});
$for M in reversed(range(MR)):
c${M} = (${XINT8_T}*) ((uintptr_t) c${M} + cn_stride);
a = (const ${XINT8_T}**restrict) ((uintptr_t) a - ks);
nc -= ${NR};
} else {
$if NR == 16:
$for M in reversed(range(MR)):
$if M % 2 == 1:
${XINT8X16_T} vout${M-1}x01234567_${M}x01234567 = ${VCOMBINE_X8}(${VGET_LOW_X8}(vout${M-1}x0123456789ABCDEF), ${VGET_LOW_X8}(vout${M}x0123456789ABCDEF));
$elif M + 1 == MR:
${XINT8X8_T} vout${M}x01234567 = ${VGET_LOW_X8}(vout${M}x0123456789ABCDEF);
if (nc & 8) {
$for M in reversed(range(MR)):
$if M % 2 == 1:
${VST1_X8}(c${M}, ${VGET_HIGH_X8}(vout${M-1}x01234567_${M}x01234567)); c${M} += 8;
${VST1_X8}(c${M-1}, ${VGET_LOW_X8}(vout${M-1}x01234567_${M}x01234567)); c${M-1} += 8;
$elif M + 1 == MR:
${VST1_X8}(c${M}, vout${M}x01234567); c${M} += 8;
$for M in reversed(range(MR)):
$if M % 2 == 1:
vout${M-1}x01234567_${M}x01234567 = ${VCOMBINE_X8}(${VGET_HIGH_X8}(vout${M-1}x0123456789ABCDEF), ${VGET_HIGH_X8}(vout${M}x0123456789ABCDEF));
$elif M + 1 == MR:
vout${M}x01234567 = ${VGET_HIGH_X8}(vout${M}x0123456789ABCDEF);
}
if (nc & 4) {
$for M in reversed(range(MR)):
$if M % 2 == 1:
vst1q_lane_u32(__builtin_assume_aligned(c${M}, 1), ${VREINTERPRETQ_U32_X8}(vout${M-1}x01234567_${M}x01234567), 2); c${M} += 4;
vst1q_lane_u32(__builtin_assume_aligned(c${M-1}, 1), ${VREINTERPRETQ_U32_X8}(vout${M-1}x01234567_${M}x01234567), 0); c${M-1} += 4;
$elif M + 1 == MR:
vst1_lane_u32(__builtin_assume_aligned(c${M}, 1), ${VREINTERPRET_U32_X8}(vout${M}x01234567), 0); c${M} += 4;
$for M in reversed(range(MR)):
$if M % 2 == 1:
vout${M-1}x01234567_${M}x01234567 = ${VEXTQ_X8}(vout${M-1}x01234567_${M}x01234567, vout${M-1}x01234567_${M}x01234567, 4);
$elif M + 1 == MR:
vout${M}x01234567 = ${VEXT_X8}(vout${M}x01234567, vout${M}x01234567, 4);
}
if (nc & 2) {
$for M in reversed(range(MR)):
$if M % 2 == 1:
vst1q_lane_u16(__builtin_assume_aligned(c${M}, 1), ${VREINTERPRETQ_U16_X8}(vout${M-1}x01234567_${M}x01234567), 4); c${M} += 2;
vst1q_lane_u16(__builtin_assume_aligned(c${M-1}, 1), ${VREINTERPRETQ_U16_X8}(vout${M-1}x01234567_${M}x01234567), 0); c${M-1} += 2;
$elif M + 1 == MR:
vst1_lane_u16(__builtin_assume_aligned(c${M}, 1), ${VREINTERPRET_U16_X8}(vout${M}x01234567), 0); c${M} += 2;
$for M in reversed(range(MR)):
$if M % 2 == 1:
vout${M-1}x01234567_${M}x01234567 = ${VEXTQ_X8}(vout${M-1}x01234567_${M}x01234567, vout${M-1}x01234567_${M}x01234567, 2);
$elif M + 1 == MR:
vout${M}x01234567 = ${VEXT_X8}(vout${M}x01234567, vout${M}x01234567, 2);
}
if (nc & 1) {
$for M in reversed(range(MR)):
$if M % 2 == 1:
${VST1Q_LANE_X8}(c${M}, vout${M-1}x01234567_${M}x01234567, 8);
${VST1Q_LANE_X8}(c${M-1}, vout${M-1}x01234567_${M}x01234567, 0);
$elif M + 1 == MR:
${VST1_LANE_X8}(c${M}, vout${M}x01234567, 0);
}
nc = 0;
}
} while (nc != 0);
}