src/qs8-vmulc/neon.c.in - platform/external/XNNPACK - Gitiles

 // Copyright 2021 Google LLC
 //
 // This source code is licensed under the BSD-style license found in the
 // LICENSE file in the root directory of this source tree.

 $assert DATATYPE in ["QS8", "QU8"]
 $assert REQUANTIZATION in ["FP32", "RNDNU"]
 $assert BATCH_TILE % (16 if LD128 else 8) == 0
 $assert BATCH_TILE >= 8
 $ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 #include <assert.h>

 #include <arm_neon.h>

 $if REQUANTIZATION == "FP32" and ARMV8:
   #include <xnnpack/intrinsics-polyfill.h>
 #include <xnnpack/vmul.h>


 $PARAMS_STRUCT = REQUANTIZATION.lower() + "_" + ("neonv8" if ARMV8 else "neon")
 $XINT8_T = {"QS8": "int8_t", "QU8": "uint8_t"}[DATATYPE]
 $XINT8X8_T = {"QS8": "int8x8_t", "QU8": "uint8x8_t"}[DATATYPE]
 $XINT8X16_T = {"QS8": "int8x16_t", "QU8": "uint8x16_t"}[DATATYPE]
 $VLD1_X8 = {"QS8": "vld1_s8", "QU8": "vld1_u8"}[DATATYPE]
 $VLD1Q_X8 = {"QS8": "vld1q_s8", "QU8": "vld1q_u8"}[DATATYPE]
 $VLD1_DUP_X8 = {"QS8": "vld1_dup_s8", "QU8": "vld1_dup_u8"}[DATATYPE]
 $VLD1Q_DUP_X8 = {"QS8": "vld1q_dup_s8", "QU8": "vld1q_dup_u8"}[DATATYPE]
 $VST1_LANE_X8 = {"QS8": "vst1_lane_s8", "QU8": "vst1_lane_u8"}[DATATYPE]
 $VST1_X8 = {"QS8": "vst1_s8", "QU8": "vst1_u8"}[DATATYPE]
 $VST1Q_X8 = {"QS8": "vst1q_s8", "QU8": "vst1q_u8"}[DATATYPE]
 $VMIN_X8 = {"QS8": "vmin_s8", "QU8": "vmin_u8"}[DATATYPE]
 $VMAX_X8 = {"QS8": "vmax_s8", "QU8": "vmax_u8"}[DATATYPE]
 $VMINQ_X8 = {"QS8": "vminq_s8", "QU8": "vminq_u8"}[DATATYPE]
 $VMAXQ_X8 = {"QS8": "vmaxq_s8", "QU8": "vmaxq_u8"}[DATATYPE]
 $VQMOVXN_S16 = {"QS8": "vqmovn_s16", "QU8": "vqmovun_s16"}[DATATYPE]
 $VQMOVXN_HIGH_S16 = {"QS8": "vqmovn_high_s16", "QU8": "vqmovun_high_s16"}[DATATYPE]
 $VEXT_X8 = {"QS8": "vext_s8", "QU8": "vext_u8"}[DATATYPE]
 $VGET_LOW_X8 = {"QS8": "vget_low_s8", "QU8": "vget_low_u8"}[DATATYPE]
 $VCOMBINE_X8 = {"QS8": "vcombine_s8", "QU8": "vcombine_u8"}[DATATYPE]
 $VREINTERPRET_U32_X8 = {"QS8": "vreinterpret_u32_s8", "QU8": "vreinterpret_u32_u8"}[DATATYPE]
 $VREINTERPRET_U16_X8 = {"QS8": "vreinterpret_u16_s8", "QU8": "vreinterpret_u16_u8"}[DATATYPE]
 void xnn_${DATATYPE.lower()}_vmulc_minmax_${REQUANTIZATION.lower()}_ukernel__${"neonv8" if ARMV8 else "neon"}_${"ld128" if LD128 else "ld64"}_x${BATCH_TILE}(
     size_t n,
     const ${XINT8_T}* input_a,
     const ${XINT8_T}* input_b,
     ${XINT8_T}* output,
     const union xnn_${DATATYPE.lower()}_mul_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
 {
   $if LD128:
     #if XNN_ARCH_ARM64
       const ${XINT8X16_T} va_zero_point = ${VLD1Q_DUP_X8}(params->${PARAMS_STRUCT}.a_zero_point);
     #else
       const ${XINT8X8_T} va_zero_point = ${VLD1_DUP_X8}(params->${PARAMS_STRUCT}.a_zero_point);
     #endif
   $else:
     const ${XINT8X8_T} va_zero_point = ${VLD1_DUP_X8}(params->${PARAMS_STRUCT}.a_zero_point);
   $if REQUANTIZATION == "FP32":
     const float32x4_t vscale = vld1q_dup_f32(&params->${PARAMS_STRUCT}.scale);
     $if ARMV8:
       const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->fp32_neonv8.output_zero_point);
     $else:
       const float32x4_t vmagic_bias = vld1q_dup_f32(&params->fp32_neon.magic_bias);
       const int32x4_t vmagic_bias_less_output_zero_point = vld1q_dup_s32(&params->fp32_neon.magic_bias_less_output_zero_point);
   $elif REQUANTIZATION == "RNDNU":
     const int32x4_t vleft_pre_shift = vld1q_dup_s32(&params->rndnu_neon.left_pre_shift);
     const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
     const int32x4_t vleft_post_shift = vld1q_dup_s32(&params->rndnu_neon.left_post_shift);
     const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
   $if BATCH_TILE > 8:
     const ${XINT8X16_T} voutput_min = ${VLD1Q_DUP_X8}(&params->${PARAMS_STRUCT}.output_min);
     const ${XINT8X16_T} voutput_max = ${VLD1Q_DUP_X8}(&params->${PARAMS_STRUCT}.output_max);
   $else:
     const ${XINT8X8_T} voutput_min = ${VLD1_DUP_X8}(&params->${PARAMS_STRUCT}.output_min);
     const ${XINT8X8_T} voutput_max = ${VLD1_DUP_X8}(&params->${PARAMS_STRUCT}.output_max);

   const ${XINT8X8_T} vb = ${VLD1_DUP_X8}(input_b);
   const ${XINT8X8_T} vb_zero_point = ${VLD1_DUP_X8}(params->${PARAMS_STRUCT}.b_zero_point);
   $if DATATYPE == "QU8":
     const int16x8_t vxb = vreinterpretq_s16_u16(vsubl_u8(vb, vb_zero_point));
   $else:
     const int16x8_t vxb = vsubl_s8(vb, vb_zero_point);
   for (; n >= ${BATCH_TILE} * sizeof(${XINT8_T}); n -= ${BATCH_TILE} * sizeof(${XINT8_T})) {
     $if LD128:
       $for N in range(0, BATCH_TILE, 16):
         const ${XINT8X16_T} va${ABC[N:N+16]} = ${VLD1Q_X8}(input_a); input_a += 16;

       #if XNN_ARCH_ARM64
         $for N in range(0, BATCH_TILE, 16):
           $if DATATYPE == "QU8":
             const int16x8_t vxa${ABC[N:N+8]} = vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(va${ABC[N:N+16]}), vget_low_u8(va_zero_point)));
             const int16x8_t vxa${ABC[N+8:N+16]} = vreinterpretq_s16_u16(vsubl_high_u8(va${ABC[N:N+16]}, va_zero_point));
           $else:
             const int16x8_t vxa${ABC[N:N+8]} = vsubl_s8(vget_low_s8(va${ABC[N:N+16]}), vget_low_s8(va_zero_point));
             const int16x8_t vxa${ABC[N+8:N+16]} = vsubl_high_s8(va${ABC[N:N+16]}, va_zero_point);
       #else  // !XNN_ARCH_ARM64
         $for N in range(0, BATCH_TILE, 16):
           $if DATATYPE == "QU8":
             const int16x8_t vxa${ABC[N:N+8]} = vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(va${ABC[N:N+16]}), va_zero_point));
             const int16x8_t vxa${ABC[N+8:N+16]} = vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(va${ABC[N:N+16]}), va_zero_point));
           $else:
             const int16x8_t vxa${ABC[N:N+8]} = vsubl_s8(vget_low_s8(va${ABC[N:N+16]}), va_zero_point);
             const int16x8_t vxa${ABC[N+8:N+16]} = vsubl_s8(vget_high_s8(va${ABC[N:N+16]}), va_zero_point);
       #endif  // XNN_ARCH_ARM64
     $else:
       $for N in range(0, BATCH_TILE, 8):
         const ${XINT8X8_T} va${ABC[N:N+8]} = ${VLD1_X8}(input_a); input_a += 8;

       $for N in range(0, BATCH_TILE, 8):
         $if DATATYPE == "QU8":
           const int16x8_t vxa${ABC[N:N+8]} = vreinterpretq_s16_u16(vsubl_u8(va${ABC[N:N+8]}, va_zero_point));
         $else:
           const int16x8_t vxa${ABC[N:N+8]} = vsubl_s8(va${ABC[N:N+8]}, va_zero_point);

     $for N in range(0, BATCH_TILE, 8):
       int32x4_t vacc${ABC[N:N+4]} = vmull_s16(vget_low_s16(vxa${ABC[N:N+8]}), vget_low_s16(vxb));
       int32x4_t vacc${ABC[N+4:N+8]} = vmull_s16(vget_high_s16(vxa${ABC[N:N+8]}), vget_high_s16(vxb));

     $if REQUANTIZATION == "FP32":
       $for N in range(0, BATCH_TILE, 4):
         float32x4_t vfpacc${ABC[N:N+4]} = vcvtq_f32_s32(vacc${ABC[N:N+4]});

       $for N in range(0, BATCH_TILE, 4):
         vfpacc${ABC[N:N+4]} = vmulq_f32(vfpacc${ABC[N:N+4]}, vscale);

       $if ARMV8:
         $for N in range(0, BATCH_TILE, 4):
           vacc${ABC[N:N+4]} = vcvtnq_s32_f32(vfpacc${ABC[N:N+4]});
       $else:
         $for N in range(0, BATCH_TILE, 4):
           vacc${ABC[N:N+4]} = vreinterpretq_s32_f32(vaddq_f32(vfpacc${ABC[N:N+4]}, vmagic_bias));

         $for N in range(0, BATCH_TILE, 4):
           vacc${ABC[N:N+4]} = vqsubq_s32(vacc${ABC[N:N+4]}, vmagic_bias_less_output_zero_point);
     $elif REQUANTIZATION == "RNDNU":
       $for N in range(0, BATCH_TILE, 4):
         vacc${ABC[N:N+4]} = vqshlq_s32(vacc${ABC[N:N+4]}, vleft_pre_shift);

       $for N in range(0, BATCH_TILE, 4):
         vacc${ABC[N:N+4]} = vqdmulhq_s32(vacc${ABC[N:N+4]}, vmultiplier);

       $for N in range(0, BATCH_TILE, 4):
         vacc${ABC[N:N+4]} = vrshlq_s32(vacc${ABC[N:N+4]}, vleft_post_shift);

     #if XNN_ARCH_ARM64
       $for N in range(0, BATCH_TILE, 8):
         int16x8_t vacc${ABC[N:N+8]} = vqmovn_high_s32(vqmovn_s32(vacc${ABC[N:N+4]}), vacc${ABC[N+4:N+8]});
     #else
       $for N in range(0, BATCH_TILE, 8):
         int16x8_t vacc${ABC[N:N+8]} = vcombine_s16(vqmovn_s32(vacc${ABC[N:N+4]}), vqmovn_s32(vacc${ABC[N+4:N+8]}));
     #endif

     $if REQUANTIZATION != "FP32" or ARMV8:
       $for N in range(0, BATCH_TILE, 8):
         vacc${ABC[N:N+8]} = vqaddq_s16(vacc${ABC[N:N+8]}, voutput_zero_point);

     #if XNN_ARCH_ARM64
       $for N in range(0, BATCH_TILE, 16):
         $if N + 8 < BATCH_TILE:
           ${XINT8X16_T} vout${ABC[N:N+16]} = ${VQMOVXN_HIGH_S16}(${VQMOVXN_S16}(vacc${ABC[N:N+8]}), vacc${ABC[N+8:N+16]});
         $else:
           ${XINT8X8_T} vout${ABC[N:N+8]} = ${VQMOVXN_S16}(vacc${ABC[N:N+8]});
     #else
       $for N in range(0, BATCH_TILE, 16):
         $if N + 8 < BATCH_TILE:
           ${XINT8X16_T} vout${ABC[N:N+16]} = ${VCOMBINE_X8}(${VQMOVXN_S16}(vacc${ABC[N:N+8]}), ${VQMOVXN_S16}(vacc${ABC[N+8:N+16]}));
         $else:
           ${XINT8X8_T} vout${ABC[N:N+8]} = ${VQMOVXN_S16}(vacc${ABC[N:N+8]});
     #endif

     $for N in range(0, BATCH_TILE, 16):
       $if N + 8 < BATCH_TILE:
         vout${ABC[N:N+16]} = ${VMAXQ_X8}(vout${ABC[N:N+16]}, voutput_min);
       $elif BATCH_TILE > 8:
         vout${ABC[N:N+8]} = ${VMAX_X8}(vout${ABC[N:N+8]}, ${VGET_LOW_X8}(voutput_min));
       $else:
         vout${ABC[N:N+8]} = ${VMAX_X8}(vout${ABC[N:N+8]}, voutput_min);

     $for N in range(0, BATCH_TILE, 16):
       $if N + 8 < BATCH_TILE:
         vout${ABC[N:N+16]} = ${VMINQ_X8}(vout${ABC[N:N+16]}, voutput_max);
       $elif BATCH_TILE > 8:
         vout${ABC[N:N+8]} = ${VMIN_X8}(vout${ABC[N:N+8]}, ${VGET_LOW_X8}(voutput_max));
       $else:
         vout${ABC[N:N+8]} = ${VMIN_X8}(vout${ABC[N:N+8]}, voutput_max);

     $for N in range(0, BATCH_TILE, 16):
       $if N + 8 < BATCH_TILE:
         ${VST1Q_X8}(output, vout${ABC[N:N+16]}); output += 16;
       $else:
         ${VST1_X8}(output, vout${ABC[N:N+8]}); output += 8;
   }
   if XNN_UNLIKELY(n != 0) {
     ${"do " if BATCH_TILE > 8 else ""}{
       $if BATCH_TILE > 8:
         const ${XINT8X8_T} va${ABC[0:8]} = ${VLD1_X8}(input_a); input_a += 8;
       $else:
         const ${XINT8X8_T} va${ABC[0:8]} = ${VLD1_X8}(input_a);

       $if LD128:
         $if DATATYPE == "QU8":
           #if XNN_ARCH_ARM64
             const int16x8_t vxa${ABC[0:8]} = vreinterpretq_s16_u16(vsubl_u8(va${ABC[0:8]}, vget_low_u8(va_zero_point)));
           #else  // !XNN_ARCH_ARM64
             const int16x8_t vxa${ABC[0:8]} = vreinterpretq_s16_u16(vsubl_u8(va${ABC[0:8]}, va_zero_point));
           #endif  // XNN_ARCH_ARM64
         $else:
           #if XNN_ARCH_ARM64
             const int16x8_t vxa${ABC[0:8]} = vsubl_s8(va${ABC[0:8]}, vget_low_s8(va_zero_point));
           #else  // !XNN_ARCH_ARM64
             const int16x8_t vxa${ABC[0:8]} = vsubl_s8(va${ABC[0:8]}, va_zero_point);
           #endif  // XNN_ARCH_ARM64
       $else:
         $if DATATYPE == "QU8":
           const int16x8_t vxa${ABC[0:8]} = vreinterpretq_s16_u16(vsubl_u8(va${ABC[0:8]}, va_zero_point));
         $else:
           const int16x8_t vxa${ABC[0:8]} = vsubl_s8(va${ABC[0:8]}, va_zero_point);

       int32x4_t vacc${ABC[0:4]} = vmull_s16(vget_low_s16(vxa${ABC[0:8]}), vget_low_s16(vxb));
       int32x4_t vacc${ABC[4:8]} = vmull_s16(vget_high_s16(vxa${ABC[0:8]}), vget_high_s16(vxb));

       $if REQUANTIZATION == "FP32":
         float32x4_t vfpacc${ABC[0:4]} = vcvtq_f32_s32(vacc${ABC[0:4]});
         float32x4_t vfpacc${ABC[4:8]} = vcvtq_f32_s32(vacc${ABC[4:8]});

         vfpacc${ABC[0:4]} = vmulq_f32(vfpacc${ABC[0:4]}, vscale);
         vfpacc${ABC[4:8]} = vmulq_f32(vfpacc${ABC[4:8]}, vscale);

         $if ARMV8:
           vacc${ABC[0:4]} = vcvtnq_s32_f32(vfpacc${ABC[0:4]});
           vacc${ABC[4:8]} = vcvtnq_s32_f32(vfpacc${ABC[4:8]});
         $else:
           vacc${ABC[0:4]} = vreinterpretq_s32_f32(vaddq_f32(vfpacc${ABC[0:4]}, vmagic_bias));
           vacc${ABC[4:8]} = vreinterpretq_s32_f32(vaddq_f32(vfpacc${ABC[4:8]}, vmagic_bias));

           vacc${ABC[0:4]} = vqsubq_s32(vacc${ABC[0:4]}, vmagic_bias_less_output_zero_point);
           vacc${ABC[4:8]} = vqsubq_s32(vacc${ABC[4:8]}, vmagic_bias_less_output_zero_point);
       $elif REQUANTIZATION == "RNDNU":
         vacc${ABC[0:4]} = vqshlq_s32(vacc${ABC[0:4]}, vleft_pre_shift);
         vacc${ABC[4:8]} = vqshlq_s32(vacc${ABC[4:8]}, vleft_pre_shift);

         vacc${ABC[0:4]} = vqdmulhq_s32(vacc${ABC[0:4]}, vmultiplier);
         vacc${ABC[4:8]} = vqdmulhq_s32(vacc${ABC[4:8]}, vmultiplier);

         vacc${ABC[0:4]} = vrshlq_s32(vacc${ABC[0:4]}, vleft_post_shift);
         vacc${ABC[4:8]} = vrshlq_s32(vacc${ABC[4:8]}, vleft_post_shift);

       #if XNN_ARCH_ARM64
         int16x8_t vacc${ABC[0:8]} = vqmovn_high_s32(vqmovn_s32(vacc${ABC[0:4]}), vacc${ABC[4:8]});
       #else
         int16x8_t vacc${ABC[0:8]} = vcombine_s16(vqmovn_s32(vacc${ABC[0:4]}), vqmovn_s32(vacc${ABC[4:8]}));
       #endif

       $if REQUANTIZATION != "FP32" or ARMV8:
         vacc${ABC[0:8]} = vqaddq_s16(vacc${ABC[0:8]}, voutput_zero_point);

       ${XINT8X8_T} vout${ABC[0:8]} = ${VQMOVXN_S16}(vacc${ABC[0:8]});

       $if BATCH_TILE > 8:
         vout${ABC[0:8]} = ${VMAX_X8}(vout${ABC[0:8]}, ${VGET_LOW_X8}(voutput_min));
         vout${ABC[0:8]} = ${VMIN_X8}(vout${ABC[0:8]}, ${VGET_LOW_X8}(voutput_max));
         if XNN_LIKELY(n >= (8 * sizeof(${XINT8_T}))) {
           ${VST1_X8}(output, vout${ABC[0:8]}); output += 8;
           n -= 8 * sizeof(${XINT8_T});
         } else {
           if (n & (4 * sizeof(${XINT8_T}))) {
             vst1_lane_u32((void*) output, ${VREINTERPRET_U32_X8}(vout${ABC[0:8]}), 0); output += 4;
             vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 4);
           }
           if (n & (2 * sizeof(${XINT8_T}))) {
             vst1_lane_u16((void*) output, ${VREINTERPRET_U16_X8}(vout${ABC[0:8]}), 0); output += 2;
             vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 2);
           }
           if (n & (1 * sizeof(${XINT8_T}))) {
             ${VST1_LANE_X8}(output, vout${ABC[0:8]}, 0);
           }
           n = 0;
         }
       $else:
         vout${ABC[0:8]} = ${VMAX_X8}(vout${ABC[0:8]}, voutput_min);
         vout${ABC[0:8]} = ${VMIN_X8}(vout${ABC[0:8]}, voutput_max);
         if (n & (4 * sizeof(${XINT8_T}))) {
           vst1_lane_u32((void*) output, ${VREINTERPRET_U32_X8}(vout${ABC[0:8]}), 0); output += 4;
           vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 4);
         }
         if (n & (2 * sizeof(${XINT8_T}))) {
           vst1_lane_u16((void*) output, ${VREINTERPRET_U16_X8}(vout${ABC[0:8]}), 0); output += 2;
           vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 2);
         }
         if (n & (1 * sizeof(${XINT8_T}))) {
           ${VST1_LANE_X8}(output, vout${ABC[0:8]}, 0);
         }
     }${" while (n != 0);" if BATCH_TILE > 8 else ""}
   }
 }
	// Copyright 2021 Google LLC
	//
	// This source code is licensed under the BSD-style license found in the
	// LICENSE file in the root directory of this source tree.

	$assert DATATYPE in ["QS8", "QU8"]
	$assert REQUANTIZATION in ["FP32", "RNDNU"]
	$assert BATCH_TILE % (16 if LD128 else 8) == 0
	$assert BATCH_TILE >= 8
	$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
	#include <assert.h>

	#include <arm_neon.h>

	$if REQUANTIZATION == "FP32" and ARMV8:
	#include <xnnpack/intrinsics-polyfill.h>
	#include <xnnpack/vmul.h>


	$PARAMS_STRUCT = REQUANTIZATION.lower() + "_" + ("neonv8" if ARMV8 else "neon")
	$XINT8_T = {"QS8": "int8_t", "QU8": "uint8_t"}[DATATYPE]
	$XINT8X8_T = {"QS8": "int8x8_t", "QU8": "uint8x8_t"}[DATATYPE]
	$XINT8X16_T = {"QS8": "int8x16_t", "QU8": "uint8x16_t"}[DATATYPE]
	$VLD1_X8 = {"QS8": "vld1_s8", "QU8": "vld1_u8"}[DATATYPE]
	$VLD1Q_X8 = {"QS8": "vld1q_s8", "QU8": "vld1q_u8"}[DATATYPE]
	$VLD1_DUP_X8 = {"QS8": "vld1_dup_s8", "QU8": "vld1_dup_u8"}[DATATYPE]
	$VLD1Q_DUP_X8 = {"QS8": "vld1q_dup_s8", "QU8": "vld1q_dup_u8"}[DATATYPE]
	$VST1_LANE_X8 = {"QS8": "vst1_lane_s8", "QU8": "vst1_lane_u8"}[DATATYPE]
	$VST1_X8 = {"QS8": "vst1_s8", "QU8": "vst1_u8"}[DATATYPE]
	$VST1Q_X8 = {"QS8": "vst1q_s8", "QU8": "vst1q_u8"}[DATATYPE]
	$VMIN_X8 = {"QS8": "vmin_s8", "QU8": "vmin_u8"}[DATATYPE]
	$VMAX_X8 = {"QS8": "vmax_s8", "QU8": "vmax_u8"}[DATATYPE]
	$VMINQ_X8 = {"QS8": "vminq_s8", "QU8": "vminq_u8"}[DATATYPE]
	$VMAXQ_X8 = {"QS8": "vmaxq_s8", "QU8": "vmaxq_u8"}[DATATYPE]
	$VQMOVXN_S16 = {"QS8": "vqmovn_s16", "QU8": "vqmovun_s16"}[DATATYPE]
	$VQMOVXN_HIGH_S16 = {"QS8": "vqmovn_high_s16", "QU8": "vqmovun_high_s16"}[DATATYPE]
	$VEXT_X8 = {"QS8": "vext_s8", "QU8": "vext_u8"}[DATATYPE]
	$VGET_LOW_X8 = {"QS8": "vget_low_s8", "QU8": "vget_low_u8"}[DATATYPE]
	$VCOMBINE_X8 = {"QS8": "vcombine_s8", "QU8": "vcombine_u8"}[DATATYPE]
	$VREINTERPRET_U32_X8 = {"QS8": "vreinterpret_u32_s8", "QU8": "vreinterpret_u32_u8"}[DATATYPE]
	$VREINTERPRET_U16_X8 = {"QS8": "vreinterpret_u16_s8", "QU8": "vreinterpret_u16_u8"}[DATATYPE]
	void xnn_${DATATYPE.lower()}_vmulc_minmax_${REQUANTIZATION.lower()}_ukernel__${"neonv8" if ARMV8 else "neon"}_${"ld128" if LD128 else "ld64"}_x${BATCH_TILE}(
	size_t n,
	const ${XINT8_T}* input_a,
	const ${XINT8_T}* input_b,
	${XINT8_T}* output,
	const union xnn_${DATATYPE.lower()}_mul_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
	{
	$if LD128:
	#if XNN_ARCH_ARM64
	const ${XINT8X16_T} va_zero_point = ${VLD1Q_DUP_X8}(params->${PARAMS_STRUCT}.a_zero_point);
	#else
	const ${XINT8X8_T} va_zero_point = ${VLD1_DUP_X8}(params->${PARAMS_STRUCT}.a_zero_point);
	#endif
	$else:
	const ${XINT8X8_T} va_zero_point = ${VLD1_DUP_X8}(params->${PARAMS_STRUCT}.a_zero_point);
	$if REQUANTIZATION == "FP32":
	const float32x4_t vscale = vld1q_dup_f32(&params->${PARAMS_STRUCT}.scale);
	$if ARMV8:
	const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->fp32_neonv8.output_zero_point);
	$else:
	const float32x4_t vmagic_bias = vld1q_dup_f32(&params->fp32_neon.magic_bias);
	const int32x4_t vmagic_bias_less_output_zero_point = vld1q_dup_s32(&params->fp32_neon.magic_bias_less_output_zero_point);
	$elif REQUANTIZATION == "RNDNU":
	const int32x4_t vleft_pre_shift = vld1q_dup_s32(&params->rndnu_neon.left_pre_shift);
	const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
	const int32x4_t vleft_post_shift = vld1q_dup_s32(&params->rndnu_neon.left_post_shift);
	const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
	$if BATCH_TILE > 8:
	const ${XINT8X16_T} voutput_min = ${VLD1Q_DUP_X8}(&params->${PARAMS_STRUCT}.output_min);
	const ${XINT8X16_T} voutput_max = ${VLD1Q_DUP_X8}(&params->${PARAMS_STRUCT}.output_max);
	$else:
	const ${XINT8X8_T} voutput_min = ${VLD1_DUP_X8}(&params->${PARAMS_STRUCT}.output_min);
	const ${XINT8X8_T} voutput_max = ${VLD1_DUP_X8}(&params->${PARAMS_STRUCT}.output_max);

	const ${XINT8X8_T} vb = ${VLD1_DUP_X8}(input_b);
	const ${XINT8X8_T} vb_zero_point = ${VLD1_DUP_X8}(params->${PARAMS_STRUCT}.b_zero_point);
	$if DATATYPE == "QU8":
	const int16x8_t vxb = vreinterpretq_s16_u16(vsubl_u8(vb, vb_zero_point));
	$else:
	const int16x8_t vxb = vsubl_s8(vb, vb_zero_point);
	for (; n >= ${BATCH_TILE} * sizeof(${XINT8_T}); n -= ${BATCH_TILE} * sizeof(${XINT8_T})) {
	$if LD128:
	$for N in range(0, BATCH_TILE, 16):
	const ${XINT8X16_T} va${ABC[N:N+16]} = ${VLD1Q_X8}(input_a); input_a += 16;

	#if XNN_ARCH_ARM64
	$for N in range(0, BATCH_TILE, 16):
	$if DATATYPE == "QU8":
	const int16x8_t vxa${ABC[N:N+8]} = vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(va${ABC[N:N+16]}), vget_low_u8(va_zero_point)));
	const int16x8_t vxa${ABC[N+8:N+16]} = vreinterpretq_s16_u16(vsubl_high_u8(va${ABC[N:N+16]}, va_zero_point));
	$else:
	const int16x8_t vxa${ABC[N:N+8]} = vsubl_s8(vget_low_s8(va${ABC[N:N+16]}), vget_low_s8(va_zero_point));
	const int16x8_t vxa${ABC[N+8:N+16]} = vsubl_high_s8(va${ABC[N:N+16]}, va_zero_point);
	#else // !XNN_ARCH_ARM64
	$for N in range(0, BATCH_TILE, 16):
	$if DATATYPE == "QU8":
	const int16x8_t vxa${ABC[N:N+8]} = vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(va${ABC[N:N+16]}), va_zero_point));
	const int16x8_t vxa${ABC[N+8:N+16]} = vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(va${ABC[N:N+16]}), va_zero_point));
	$else:
	const int16x8_t vxa${ABC[N:N+8]} = vsubl_s8(vget_low_s8(va${ABC[N:N+16]}), va_zero_point);
	const int16x8_t vxa${ABC[N+8:N+16]} = vsubl_s8(vget_high_s8(va${ABC[N:N+16]}), va_zero_point);
	#endif // XNN_ARCH_ARM64
	$else:
	$for N in range(0, BATCH_TILE, 8):
	const ${XINT8X8_T} va${ABC[N:N+8]} = ${VLD1_X8}(input_a); input_a += 8;

	$for N in range(0, BATCH_TILE, 8):
	$if DATATYPE == "QU8":
	const int16x8_t vxa${ABC[N:N+8]} = vreinterpretq_s16_u16(vsubl_u8(va${ABC[N:N+8]}, va_zero_point));
	$else:
	const int16x8_t vxa${ABC[N:N+8]} = vsubl_s8(va${ABC[N:N+8]}, va_zero_point);

	$for N in range(0, BATCH_TILE, 8):
	int32x4_t vacc${ABC[N:N+4]} = vmull_s16(vget_low_s16(vxa${ABC[N:N+8]}), vget_low_s16(vxb));
	int32x4_t vacc${ABC[N+4:N+8]} = vmull_s16(vget_high_s16(vxa${ABC[N:N+8]}), vget_high_s16(vxb));

	$if REQUANTIZATION == "FP32":
	$for N in range(0, BATCH_TILE, 4):
	float32x4_t vfpacc${ABC[N:N+4]} = vcvtq_f32_s32(vacc${ABC[N:N+4]});

	$for N in range(0, BATCH_TILE, 4):
	vfpacc${ABC[N:N+4]} = vmulq_f32(vfpacc${ABC[N:N+4]}, vscale);

	$if ARMV8:
	$for N in range(0, BATCH_TILE, 4):
	vacc${ABC[N:N+4]} = vcvtnq_s32_f32(vfpacc${ABC[N:N+4]});
	$else:
	$for N in range(0, BATCH_TILE, 4):
	vacc${ABC[N:N+4]} = vreinterpretq_s32_f32(vaddq_f32(vfpacc${ABC[N:N+4]}, vmagic_bias));

	$for N in range(0, BATCH_TILE, 4):
	vacc${ABC[N:N+4]} = vqsubq_s32(vacc${ABC[N:N+4]}, vmagic_bias_less_output_zero_point);
	$elif REQUANTIZATION == "RNDNU":
	$for N in range(0, BATCH_TILE, 4):
	vacc${ABC[N:N+4]} = vqshlq_s32(vacc${ABC[N:N+4]}, vleft_pre_shift);

	$for N in range(0, BATCH_TILE, 4):
	vacc${ABC[N:N+4]} = vqdmulhq_s32(vacc${ABC[N:N+4]}, vmultiplier);

	$for N in range(0, BATCH_TILE, 4):
	vacc${ABC[N:N+4]} = vrshlq_s32(vacc${ABC[N:N+4]}, vleft_post_shift);

	#if XNN_ARCH_ARM64
	$for N in range(0, BATCH_TILE, 8):
	int16x8_t vacc${ABC[N:N+8]} = vqmovn_high_s32(vqmovn_s32(vacc${ABC[N:N+4]}), vacc${ABC[N+4:N+8]});
	#else
	$for N in range(0, BATCH_TILE, 8):
	int16x8_t vacc${ABC[N:N+8]} = vcombine_s16(vqmovn_s32(vacc${ABC[N:N+4]}), vqmovn_s32(vacc${ABC[N+4:N+8]}));
	#endif

	$if REQUANTIZATION != "FP32" or ARMV8:
	$for N in range(0, BATCH_TILE, 8):
	vacc${ABC[N:N+8]} = vqaddq_s16(vacc${ABC[N:N+8]}, voutput_zero_point);

	#if XNN_ARCH_ARM64
	$for N in range(0, BATCH_TILE, 16):
	$if N + 8 < BATCH_TILE:
	${XINT8X16_T} vout${ABC[N:N+16]} = ${VQMOVXN_HIGH_S16}(${VQMOVXN_S16}(vacc${ABC[N:N+8]}), vacc${ABC[N+8:N+16]});
	$else:
	${XINT8X8_T} vout${ABC[N:N+8]} = ${VQMOVXN_S16}(vacc${ABC[N:N+8]});
	#else
	$for N in range(0, BATCH_TILE, 16):
	$if N + 8 < BATCH_TILE:
	${XINT8X16_T} vout${ABC[N:N+16]} = ${VCOMBINE_X8}(${VQMOVXN_S16}(vacc${ABC[N:N+8]}), ${VQMOVXN_S16}(vacc${ABC[N+8:N+16]}));
	$else:
	${XINT8X8_T} vout${ABC[N:N+8]} = ${VQMOVXN_S16}(vacc${ABC[N:N+8]});
	#endif

	$for N in range(0, BATCH_TILE, 16):
	$if N + 8 < BATCH_TILE:
	vout${ABC[N:N+16]} = ${VMAXQ_X8}(vout${ABC[N:N+16]}, voutput_min);
	$elif BATCH_TILE > 8:
	vout${ABC[N:N+8]} = ${VMAX_X8}(vout${ABC[N:N+8]}, ${VGET_LOW_X8}(voutput_min));
	$else:
	vout${ABC[N:N+8]} = ${VMAX_X8}(vout${ABC[N:N+8]}, voutput_min);

	$for N in range(0, BATCH_TILE, 16):
	$if N + 8 < BATCH_TILE:
	vout${ABC[N:N+16]} = ${VMINQ_X8}(vout${ABC[N:N+16]}, voutput_max);
	$elif BATCH_TILE > 8:
	vout${ABC[N:N+8]} = ${VMIN_X8}(vout${ABC[N:N+8]}, ${VGET_LOW_X8}(voutput_max));
	$else:
	vout${ABC[N:N+8]} = ${VMIN_X8}(vout${ABC[N:N+8]}, voutput_max);

	$for N in range(0, BATCH_TILE, 16):
	$if N + 8 < BATCH_TILE:
	${VST1Q_X8}(output, vout${ABC[N:N+16]}); output += 16;
	$else:
	${VST1_X8}(output, vout${ABC[N:N+8]}); output += 8;
	}
	if XNN_UNLIKELY(n != 0) {
	${"do " if BATCH_TILE > 8 else ""}{
	$if BATCH_TILE > 8:
	const ${XINT8X8_T} va${ABC[0:8]} = ${VLD1_X8}(input_a); input_a += 8;
	$else:
	const ${XINT8X8_T} va${ABC[0:8]} = ${VLD1_X8}(input_a);

	$if LD128:
	$if DATATYPE == "QU8":
	#if XNN_ARCH_ARM64
	const int16x8_t vxa${ABC[0:8]} = vreinterpretq_s16_u16(vsubl_u8(va${ABC[0:8]}, vget_low_u8(va_zero_point)));
	#else // !XNN_ARCH_ARM64
	const int16x8_t vxa${ABC[0:8]} = vreinterpretq_s16_u16(vsubl_u8(va${ABC[0:8]}, va_zero_point));
	#endif // XNN_ARCH_ARM64
	$else:
	#if XNN_ARCH_ARM64
	const int16x8_t vxa${ABC[0:8]} = vsubl_s8(va${ABC[0:8]}, vget_low_s8(va_zero_point));
	#else // !XNN_ARCH_ARM64
	const int16x8_t vxa${ABC[0:8]} = vsubl_s8(va${ABC[0:8]}, va_zero_point);
	#endif // XNN_ARCH_ARM64
	$else:
	$if DATATYPE == "QU8":
	const int16x8_t vxa${ABC[0:8]} = vreinterpretq_s16_u16(vsubl_u8(va${ABC[0:8]}, va_zero_point));
	$else:
	const int16x8_t vxa${ABC[0:8]} = vsubl_s8(va${ABC[0:8]}, va_zero_point);

	int32x4_t vacc${ABC[0:4]} = vmull_s16(vget_low_s16(vxa${ABC[0:8]}), vget_low_s16(vxb));
	int32x4_t vacc${ABC[4:8]} = vmull_s16(vget_high_s16(vxa${ABC[0:8]}), vget_high_s16(vxb));

	$if REQUANTIZATION == "FP32":
	float32x4_t vfpacc${ABC[0:4]} = vcvtq_f32_s32(vacc${ABC[0:4]});
	float32x4_t vfpacc${ABC[4:8]} = vcvtq_f32_s32(vacc${ABC[4:8]});

	vfpacc${ABC[0:4]} = vmulq_f32(vfpacc${ABC[0:4]}, vscale);
	vfpacc${ABC[4:8]} = vmulq_f32(vfpacc${ABC[4:8]}, vscale);

	$if ARMV8:
	vacc${ABC[0:4]} = vcvtnq_s32_f32(vfpacc${ABC[0:4]});
	vacc${ABC[4:8]} = vcvtnq_s32_f32(vfpacc${ABC[4:8]});
	$else:
	vacc${ABC[0:4]} = vreinterpretq_s32_f32(vaddq_f32(vfpacc${ABC[0:4]}, vmagic_bias));
	vacc${ABC[4:8]} = vreinterpretq_s32_f32(vaddq_f32(vfpacc${ABC[4:8]}, vmagic_bias));

	vacc${ABC[0:4]} = vqsubq_s32(vacc${ABC[0:4]}, vmagic_bias_less_output_zero_point);
	vacc${ABC[4:8]} = vqsubq_s32(vacc${ABC[4:8]}, vmagic_bias_less_output_zero_point);
	$elif REQUANTIZATION == "RNDNU":
	vacc${ABC[0:4]} = vqshlq_s32(vacc${ABC[0:4]}, vleft_pre_shift);
	vacc${ABC[4:8]} = vqshlq_s32(vacc${ABC[4:8]}, vleft_pre_shift);

	vacc${ABC[0:4]} = vqdmulhq_s32(vacc${ABC[0:4]}, vmultiplier);
	vacc${ABC[4:8]} = vqdmulhq_s32(vacc${ABC[4:8]}, vmultiplier);

	vacc${ABC[0:4]} = vrshlq_s32(vacc${ABC[0:4]}, vleft_post_shift);
	vacc${ABC[4:8]} = vrshlq_s32(vacc${ABC[4:8]}, vleft_post_shift);

	#if XNN_ARCH_ARM64
	int16x8_t vacc${ABC[0:8]} = vqmovn_high_s32(vqmovn_s32(vacc${ABC[0:4]}), vacc${ABC[4:8]});
	#else
	int16x8_t vacc${ABC[0:8]} = vcombine_s16(vqmovn_s32(vacc${ABC[0:4]}), vqmovn_s32(vacc${ABC[4:8]}));
	#endif

	$if REQUANTIZATION != "FP32" or ARMV8:
	vacc${ABC[0:8]} = vqaddq_s16(vacc${ABC[0:8]}, voutput_zero_point);

	${XINT8X8_T} vout${ABC[0:8]} = ${VQMOVXN_S16}(vacc${ABC[0:8]});

	$if BATCH_TILE > 8:
	vout${ABC[0:8]} = ${VMAX_X8}(vout${ABC[0:8]}, ${VGET_LOW_X8}(voutput_min));
	vout${ABC[0:8]} = ${VMIN_X8}(vout${ABC[0:8]}, ${VGET_LOW_X8}(voutput_max));
	if XNN_LIKELY(n >= (8 * sizeof(${XINT8_T}))) {
	${VST1_X8}(output, vout${ABC[0:8]}); output += 8;
	n -= 8 * sizeof(${XINT8_T});
	} else {
	if (n & (4 * sizeof(${XINT8_T}))) {
	vst1_lane_u32((void*) output, ${VREINTERPRET_U32_X8}(vout${ABC[0:8]}), 0); output += 4;
	vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 4);
	}
	if (n & (2 * sizeof(${XINT8_T}))) {
	vst1_lane_u16((void*) output, ${VREINTERPRET_U16_X8}(vout${ABC[0:8]}), 0); output += 2;
	vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 2);
	}
	if (n & (1 * sizeof(${XINT8_T}))) {
	${VST1_LANE_X8}(output, vout${ABC[0:8]}, 0);
	}
	n = 0;
	}
	$else:
	vout${ABC[0:8]} = ${VMAX_X8}(vout${ABC[0:8]}, voutput_min);
	vout${ABC[0:8]} = ${VMIN_X8}(vout${ABC[0:8]}, voutput_max);
	if (n & (4 * sizeof(${XINT8_T}))) {
	vst1_lane_u32((void*) output, ${VREINTERPRET_U32_X8}(vout${ABC[0:8]}), 0); output += 4;
	vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 4);
	}
	if (n & (2 * sizeof(${XINT8_T}))) {
	vst1_lane_u16((void*) output, ${VREINTERPRET_U16_X8}(vout${ABC[0:8]}), 0); output += 2;
	vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 2);
	}
	if (n & (1 * sizeof(${XINT8_T}))) {
	${VST1_LANE_X8}(output, vout${ABC[0:8]}, 0);
	}
	}${" while (n != 0);" if BATCH_TILE > 8 else ""}
	}
	}