Minor optimization for AArch64 NEON QS8 microkernels with FP32 requantization
PiperOrigin-RevId: 381883245
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-fp32-neon-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-fp32-neon-mul16.c
index 1b2cf39..7dca15d 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-fp32-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-fp32-neon-mul16.c
@@ -452,10 +452,10 @@
vaccCDEF = vsubq_s32(vaccCDEF, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
- const int16x8_t vacc89ABCDEF = vmovn_high_s32(vmovn_s32(vacc89AB), vaccCDEF);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
+ const int16x8_t vacc89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc89AB), vreinterpretq_s16_s32(vaccCDEF));
- int8x16_t vout0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc01234567), vacc89ABCDEF);
+ int8x16_t vout0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc01234567), vreinterpretq_s8_s16(vacc89ABCDEF));
#else
const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
const int16x8_t vacc89ABCDEF = vcombine_s16(vmovn_s32(vacc89AB), vmovn_s32(vaccCDEF));
@@ -617,7 +617,7 @@
vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
int8x8_t vout01234567 = vmovn_s16(vacc01234567);
#else
const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-fp32-neon-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-fp32-neon-mul16.c
index ad8d6d9..643deab 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-fp32-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-fp32-neon-mul16.c
@@ -212,10 +212,10 @@
vaccCDEF = vsubq_s32(vaccCDEF, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
- const int16x8_t vacc89ABCDEF = vmovn_high_s32(vmovn_s32(vacc89AB), vaccCDEF);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
+ const int16x8_t vacc89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc89AB), vreinterpretq_s16_s32(vaccCDEF));
- int8x16_t vout0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc01234567), vacc89ABCDEF);
+ int8x16_t vout0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc01234567), vreinterpretq_s8_s16(vacc89ABCDEF));
#else
const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
const int16x8_t vacc89ABCDEF = vcombine_s16(vmovn_s32(vacc89AB), vmovn_s32(vaccCDEF));
@@ -297,7 +297,7 @@
vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
int8x8_t vout01234567 = vmovn_s16(vacc01234567);
#else
const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-fp32-neon-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-fp32-neon-mul16.c
index 4349b13..b1c5526 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-fp32-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-fp32-neon-mul16.c
@@ -566,11 +566,11 @@
vaccKLMN = vsubq_s32(vaccKLMN, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
- const int16x8_t vacc89ABCDEF = vmovn_high_s32(vmovn_s32(vacc89AB), vaccCDEF);
- const int16x8_t vaccGHIJKLMN = vmovn_high_s32(vmovn_s32(vaccGHIJ), vaccKLMN);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
+ const int16x8_t vacc89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc89AB), vreinterpretq_s16_s32(vaccCDEF));
+ const int16x8_t vaccGHIJKLMN = vuzp1q_s16(vreinterpretq_s16_s32(vaccGHIJ), vreinterpretq_s16_s32(vaccKLMN));
- int8x16_t vout0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc01234567), vacc89ABCDEF);
+ int8x16_t vout0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc01234567), vreinterpretq_s8_s16(vacc89ABCDEF));
int8x8_t voutGHIJKLMN = vmovn_s16(vaccGHIJKLMN);
#else
const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
@@ -736,7 +736,7 @@
vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
int8x8_t vout01234567 = vmovn_s16(vacc01234567);
#else
const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-fp32-neon-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-fp32-neon-mul16.c
index af16153..0acf8e1 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-fp32-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-fp32-neon-mul16.c
@@ -262,11 +262,11 @@
vaccKLMN = vsubq_s32(vaccKLMN, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
- const int16x8_t vacc89ABCDEF = vmovn_high_s32(vmovn_s32(vacc89AB), vaccCDEF);
- const int16x8_t vaccGHIJKLMN = vmovn_high_s32(vmovn_s32(vaccGHIJ), vaccKLMN);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
+ const int16x8_t vacc89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc89AB), vreinterpretq_s16_s32(vaccCDEF));
+ const int16x8_t vaccGHIJKLMN = vuzp1q_s16(vreinterpretq_s16_s32(vaccGHIJ), vreinterpretq_s16_s32(vaccKLMN));
- int8x16_t vout0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc01234567), vacc89ABCDEF);
+ int8x16_t vout0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc01234567), vreinterpretq_s8_s16(vacc89ABCDEF));
int8x8_t voutGHIJKLMN = vmovn_s16(vaccGHIJKLMN);
#else
const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
@@ -352,7 +352,7 @@
vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
int8x8_t vout01234567 = vmovn_s16(vacc01234567);
#else
const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
diff --git a/src/qs8-dwconv/gen/up32x25-minmax-fp32-neon-mul16.c b/src/qs8-dwconv/gen/up32x25-minmax-fp32-neon-mul16.c
index 813c8ed..e4300a4 100644
--- a/src/qs8-dwconv/gen/up32x25-minmax-fp32-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up32x25-minmax-fp32-neon-mul16.c
@@ -680,13 +680,13 @@
vaccSTUV = vsubq_s32(vaccSTUV, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
- const int16x8_t vacc89ABCDEF = vmovn_high_s32(vmovn_s32(vacc89AB), vaccCDEF);
- const int16x8_t vaccGHIJKLMN = vmovn_high_s32(vmovn_s32(vaccGHIJ), vaccKLMN);
- const int16x8_t vaccOPQRSTUV = vmovn_high_s32(vmovn_s32(vaccOPQR), vaccSTUV);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
+ const int16x8_t vacc89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc89AB), vreinterpretq_s16_s32(vaccCDEF));
+ const int16x8_t vaccGHIJKLMN = vuzp1q_s16(vreinterpretq_s16_s32(vaccGHIJ), vreinterpretq_s16_s32(vaccKLMN));
+ const int16x8_t vaccOPQRSTUV = vuzp1q_s16(vreinterpretq_s16_s32(vaccOPQR), vreinterpretq_s16_s32(vaccSTUV));
- int8x16_t vout0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc01234567), vacc89ABCDEF);
- int8x16_t voutGHIJKLMNOPQRSTUV = vmovn_high_s16(vmovn_s16(vaccGHIJKLMN), vaccOPQRSTUV);
+ int8x16_t vout0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc01234567), vreinterpretq_s8_s16(vacc89ABCDEF));
+ int8x16_t voutGHIJKLMNOPQRSTUV = vuzp1q_s8(vreinterpretq_s8_s16(vaccGHIJKLMN), vreinterpretq_s8_s16(vaccOPQRSTUV));
#else
const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
const int16x8_t vacc89ABCDEF = vcombine_s16(vmovn_s32(vacc89AB), vmovn_s32(vaccCDEF));
@@ -852,7 +852,7 @@
vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
int8x8_t vout01234567 = vmovn_s16(vacc01234567);
#else
const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
diff --git a/src/qs8-dwconv/gen/up32x9-minmax-fp32-neon-mul16.c b/src/qs8-dwconv/gen/up32x9-minmax-fp32-neon-mul16.c
index a3a1fb4..68026e3 100644
--- a/src/qs8-dwconv/gen/up32x9-minmax-fp32-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up32x9-minmax-fp32-neon-mul16.c
@@ -312,13 +312,13 @@
vaccSTUV = vsubq_s32(vaccSTUV, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
- const int16x8_t vacc89ABCDEF = vmovn_high_s32(vmovn_s32(vacc89AB), vaccCDEF);
- const int16x8_t vaccGHIJKLMN = vmovn_high_s32(vmovn_s32(vaccGHIJ), vaccKLMN);
- const int16x8_t vaccOPQRSTUV = vmovn_high_s32(vmovn_s32(vaccOPQR), vaccSTUV);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
+ const int16x8_t vacc89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc89AB), vreinterpretq_s16_s32(vaccCDEF));
+ const int16x8_t vaccGHIJKLMN = vuzp1q_s16(vreinterpretq_s16_s32(vaccGHIJ), vreinterpretq_s16_s32(vaccKLMN));
+ const int16x8_t vaccOPQRSTUV = vuzp1q_s16(vreinterpretq_s16_s32(vaccOPQR), vreinterpretq_s16_s32(vaccSTUV));
- int8x16_t vout0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc01234567), vacc89ABCDEF);
- int8x16_t voutGHIJKLMNOPQRSTUV = vmovn_high_s16(vmovn_s16(vaccGHIJKLMN), vaccOPQRSTUV);
+ int8x16_t vout0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc01234567), vreinterpretq_s8_s16(vacc89ABCDEF));
+ int8x16_t voutGHIJKLMNOPQRSTUV = vuzp1q_s8(vreinterpretq_s8_s16(vaccGHIJKLMN), vreinterpretq_s8_s16(vaccOPQRSTUV));
#else
const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
const int16x8_t vacc89ABCDEF = vcombine_s16(vmovn_s32(vacc89AB), vmovn_s32(vaccCDEF));
@@ -404,7 +404,7 @@
vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
int8x8_t vout01234567 = vmovn_s16(vacc01234567);
#else
const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-fp32-neon-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-fp32-neon-mul16.c
index f23d668..ff697c1 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-fp32-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-fp32-neon-mul16.c
@@ -338,7 +338,7 @@
vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
int8x8_t vout01234567 = vmovn_s16(vacc01234567);
#else
@@ -500,7 +500,7 @@
vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
int8x8_t vout01234567 = vmovn_s16(vacc01234567);
#else
const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-fp32-neon-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-fp32-neon-mul16.c
index f1d8aaf..a4055d5 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-fp32-neon-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-fp32-neon-mul16.c
@@ -162,7 +162,7 @@
vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
int8x8_t vout01234567 = vmovn_s16(vacc01234567);
#else
@@ -244,7 +244,7 @@
vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc01234567 = vmovn_high_s32(vmovn_s32(vacc0123), vacc4567);
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
int8x8_t vout01234567 = vmovn_s16(vacc01234567);
#else
const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
diff --git a/src/qs8-dwconv/unipass-neon-mul16.c.in b/src/qs8-dwconv/unipass-neon-mul16.c.in
index a15f7df..803b749 100644
--- a/src/qs8-dwconv/unipass-neon-mul16.c.in
+++ b/src/qs8-dwconv/unipass-neon-mul16.c.in
@@ -111,11 +111,11 @@
#if XNN_ARCH_ARM64
$if REQUANTIZATION == "FP32" and not ARMV8:
$for C in range(0, CHANNEL_TILE, 8):
- const int16x8_t vacc${ABC[C:C+8]} = vmovn_high_s32(vmovn_s32(vacc${ABC[C:C+4]}), vacc${ABC[C+4:C+8]});
+ const int16x8_t vacc${ABC[C:C+8]} = vuzp1q_s16(vreinterpretq_s16_s32(vacc${ABC[C:C+4]}), vreinterpretq_s16_s32(vacc${ABC[C+4:C+8]}));
$for C in range(0, CHANNEL_TILE, 16):
$if C + 8 < CHANNEL_TILE:
- int8x16_t vout${ABC[C:C+16]} = vmovn_high_s16(vmovn_s16(vacc${ABC[C:C+8]}), vacc${ABC[C+8:C+16]});
+ int8x16_t vout${ABC[C:C+16]} = vuzp1q_s8(vreinterpretq_s8_s16(vacc${ABC[C:C+8]}), vreinterpretq_s8_s16(vacc${ABC[C+8:C+16]}));
$else:
int8x8_t vout${ABC[C:C+8]} = vmovn_s16(vacc${ABC[C:C+8]});
$else:
@@ -233,7 +233,7 @@
#if XNN_ARCH_ARM64
$if REQUANTIZATION == "FP32" and not ARMV8:
- const int16x8_t vacc${ABC[0:8]} = vmovn_high_s32(vmovn_s32(vacc${ABC[0:4]}), vacc${ABC[4:8]});
+ const int16x8_t vacc${ABC[0:8]} = vuzp1q_s16(vreinterpretq_s16_s32(vacc${ABC[0:4]}), vreinterpretq_s16_s32(vacc${ABC[4:8]}));
int8x8_t vout${ABC[0:8]} = vmovn_s16(vacc${ABC[0:8]});
$else:
const int16x8_t vacc${ABC[0:8]} = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc${ABC[0:4]}), vacc${ABC[4:8]}), voutput_zero_point);
diff --git a/src/qs8-gemm/c8-neon-mull-padal.c.in b/src/qs8-gemm/c8-neon-mull-padal.c.in
index 57bfbe3..227bd31 100644
--- a/src/qs8-gemm/c8-neon-mull-padal.c.in
+++ b/src/qs8-gemm/c8-neon-mull-padal.c.in
@@ -181,14 +181,14 @@
$if REQUANTIZATION == "FP32" and not ARMV8:
$for M in range(MR):
$for N in range(0, NR, 8):
- const int16x8_t vacc${M}x${ABC[N:N+8]} = vmovn_high_s32(vmovn_s32(vacc${M}x${ABC[N:N+4]}), vacc${M}x${ABC[N+4:N+8]});
+ const int16x8_t vacc${M}x${ABC[N:N+8]} = vuzp1q_s16(vreinterpretq_s16_s32(vacc${M}x${ABC[N:N+4]}), vreinterpretq_s16_s32(vacc${M}x${ABC[N+4:N+8]}));
$for M in range(MR):
$for N in range(0, NR, 16):
$if N + 8 < NR:
- int8x16_t vout${M}x${ABC[N:N+16]} = vmovn_high_s16(vmovn_s16(vacc${M}x${ABC[N:N+8]}), vacc${M}x${ABC[N+8:N+16]});
+ int8x16_t vout${M}x${ABC[N:N+16]} = vuzp1q_s8(vreinterpretq_s8_s16(vacc${M}x${ABC[N:N+8]}), vreinterpretq_s8_s16(vacc${M}x${ABC[N+8:N+16]}));
$elif M % 2 == 1:
- int8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vmovn_high_s16(vmovn_s16(vacc${M-1}x${ABC[N:N+8]}), vacc${M}x${ABC[N:N+8]});
+ int8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vuzp1q_s8(vreinterpretq_s8_s16(vacc${M-1}x${ABC[N:N+8]}), vreinterpretq_s8_s16(vacc${M}x${ABC[N:N+8]}));
$elif M + 1 == MR:
int8x8_t vout${M}x${ABC[N:N+8]} = vmovn_s16(vacc${M}x${ABC[N:N+8]});
$else:
diff --git a/src/qs8-gemm/gen/1x16-minmax-fp32-neon-mlal-lane.c b/src/qs8-gemm/gen/1x16-minmax-fp32-neon-mlal-lane.c
index be4dddc..eca8ef2 100644
--- a/src/qs8-gemm/gen/1x16-minmax-fp32-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/1x16-minmax-fp32-neon-mlal-lane.c
@@ -258,10 +258,10 @@
vacc0xCDEF = vsubq_s32(vacc0xCDEF, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc0x01234567 = vmovn_high_s32(vmovn_s32(vacc0x0123), vacc0x4567);
- const int16x8_t vacc0x89ABCDEF = vmovn_high_s32(vmovn_s32(vacc0x89AB), vacc0xCDEF);
+ const int16x8_t vacc0x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0x0123), vreinterpretq_s16_s32(vacc0x4567));
+ const int16x8_t vacc0x89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc0x89AB), vreinterpretq_s16_s32(vacc0xCDEF));
- int8x16_t vout0x0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc0x01234567), vacc0x89ABCDEF);
+ int8x16_t vout0x0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc0x01234567), vreinterpretq_s8_s16(vacc0x89ABCDEF));
#else
const int16x8_t vacc0x01234567 = vcombine_s16(vmovn_s32(vacc0x0123), vmovn_s32(vacc0x4567));
const int16x8_t vacc0x89ABCDEF = vcombine_s16(vmovn_s32(vacc0x89AB), vmovn_s32(vacc0xCDEF));
diff --git a/src/qs8-gemm/gen/1x8c8-minmax-fp32-neon-mlal-padal.c b/src/qs8-gemm/gen/1x8c8-minmax-fp32-neon-mlal-padal.c
index 19c3f8b..7acbfed 100644
--- a/src/qs8-gemm/gen/1x8c8-minmax-fp32-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/1x8c8-minmax-fp32-neon-mlal-padal.c
@@ -181,7 +181,7 @@
vacc0x4567 = vsubq_s32(vacc0x4567, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc0x01234567 = vmovn_high_s32(vmovn_s32(vacc0x0123), vacc0x4567);
+ const int16x8_t vacc0x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0x0123), vreinterpretq_s16_s32(vacc0x4567));
int8x8_t vout0x01234567 = vmovn_s16(vacc0x01234567);
#else
diff --git a/src/qs8-gemm/gen/2x8c8-minmax-fp32-neon-mlal-padal.c b/src/qs8-gemm/gen/2x8c8-minmax-fp32-neon-mlal-padal.c
index ca95762..2e62caa 100644
--- a/src/qs8-gemm/gen/2x8c8-minmax-fp32-neon-mlal-padal.c
+++ b/src/qs8-gemm/gen/2x8c8-minmax-fp32-neon-mlal-padal.c
@@ -270,10 +270,10 @@
vacc1x4567 = vsubq_s32(vacc1x4567, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc0x01234567 = vmovn_high_s32(vmovn_s32(vacc0x0123), vacc0x4567);
- const int16x8_t vacc1x01234567 = vmovn_high_s32(vmovn_s32(vacc1x0123), vacc1x4567);
+ const int16x8_t vacc0x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0x0123), vreinterpretq_s16_s32(vacc0x4567));
+ const int16x8_t vacc1x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc1x0123), vreinterpretq_s16_s32(vacc1x4567));
- int8x16_t vout0x01234567_1x01234567 = vmovn_high_s16(vmovn_s16(vacc0x01234567), vacc1x01234567);
+ int8x16_t vout0x01234567_1x01234567 = vuzp1q_s8(vreinterpretq_s8_s16(vacc0x01234567), vreinterpretq_s8_s16(vacc1x01234567));
#else
const int16x8_t vacc0x01234567 = vcombine_s16(vmovn_s32(vacc0x0123), vmovn_s32(vacc0x4567));
const int16x8_t vacc1x01234567 = vcombine_s16(vmovn_s32(vacc1x0123), vmovn_s32(vacc1x4567));
diff --git a/src/qs8-gemm/gen/4x16-minmax-fp32-neon-mlal-lane.c b/src/qs8-gemm/gen/4x16-minmax-fp32-neon-mlal-lane.c
index 94bc5b8..f67b575 100644
--- a/src/qs8-gemm/gen/4x16-minmax-fp32-neon-mlal-lane.c
+++ b/src/qs8-gemm/gen/4x16-minmax-fp32-neon-mlal-lane.c
@@ -552,19 +552,19 @@
vacc3xCDEF = vsubq_s32(vacc3xCDEF, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc0x01234567 = vmovn_high_s32(vmovn_s32(vacc0x0123), vacc0x4567);
- const int16x8_t vacc0x89ABCDEF = vmovn_high_s32(vmovn_s32(vacc0x89AB), vacc0xCDEF);
- const int16x8_t vacc1x01234567 = vmovn_high_s32(vmovn_s32(vacc1x0123), vacc1x4567);
- const int16x8_t vacc1x89ABCDEF = vmovn_high_s32(vmovn_s32(vacc1x89AB), vacc1xCDEF);
- const int16x8_t vacc2x01234567 = vmovn_high_s32(vmovn_s32(vacc2x0123), vacc2x4567);
- const int16x8_t vacc2x89ABCDEF = vmovn_high_s32(vmovn_s32(vacc2x89AB), vacc2xCDEF);
- const int16x8_t vacc3x01234567 = vmovn_high_s32(vmovn_s32(vacc3x0123), vacc3x4567);
- const int16x8_t vacc3x89ABCDEF = vmovn_high_s32(vmovn_s32(vacc3x89AB), vacc3xCDEF);
+ const int16x8_t vacc0x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0x0123), vreinterpretq_s16_s32(vacc0x4567));
+ const int16x8_t vacc0x89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc0x89AB), vreinterpretq_s16_s32(vacc0xCDEF));
+ const int16x8_t vacc1x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc1x0123), vreinterpretq_s16_s32(vacc1x4567));
+ const int16x8_t vacc1x89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc1x89AB), vreinterpretq_s16_s32(vacc1xCDEF));
+ const int16x8_t vacc2x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc2x0123), vreinterpretq_s16_s32(vacc2x4567));
+ const int16x8_t vacc2x89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc2x89AB), vreinterpretq_s16_s32(vacc2xCDEF));
+ const int16x8_t vacc3x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc3x0123), vreinterpretq_s16_s32(vacc3x4567));
+ const int16x8_t vacc3x89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc3x89AB), vreinterpretq_s16_s32(vacc3xCDEF));
- int8x16_t vout0x0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc0x01234567), vacc0x89ABCDEF);
- int8x16_t vout1x0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc1x01234567), vacc1x89ABCDEF);
- int8x16_t vout2x0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc2x01234567), vacc2x89ABCDEF);
- int8x16_t vout3x0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc3x01234567), vacc3x89ABCDEF);
+ int8x16_t vout0x0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc0x01234567), vreinterpretq_s8_s16(vacc0x89ABCDEF));
+ int8x16_t vout1x0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc1x01234567), vreinterpretq_s8_s16(vacc1x89ABCDEF));
+ int8x16_t vout2x0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc2x01234567), vreinterpretq_s8_s16(vacc2x89ABCDEF));
+ int8x16_t vout3x0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc3x01234567), vreinterpretq_s8_s16(vacc3x89ABCDEF));
#else
const int16x8_t vacc0x01234567 = vcombine_s16(vmovn_s32(vacc0x0123), vmovn_s32(vacc0x4567));
const int16x8_t vacc0x89ABCDEF = vcombine_s16(vmovn_s32(vacc0x89AB), vmovn_s32(vacc0xCDEF));
diff --git a/src/qs8-gemm/neon-mlal-lane.c.in b/src/qs8-gemm/neon-mlal-lane.c.in
index d3a30cb..943c9f6 100644
--- a/src/qs8-gemm/neon-mlal-lane.c.in
+++ b/src/qs8-gemm/neon-mlal-lane.c.in
@@ -233,14 +233,14 @@
$if REQUANTIZATION == "FP32" and not ARMV8:
$for M in range(MR):
$for N in range(0, NR, 8):
- const int16x8_t vacc${M}x${ABC[N:N+8]} = vmovn_high_s32(vmovn_s32(vacc${M}x${ABC[N:N+4]}), vacc${M}x${ABC[N+4:N+8]});
+ const int16x8_t vacc${M}x${ABC[N:N+8]} = vuzp1q_s16(vreinterpretq_s16_s32(vacc${M}x${ABC[N:N+4]}), vreinterpretq_s16_s32(vacc${M}x${ABC[N+4:N+8]}));
$for M in range(MR):
$for N in range(0, NR, 16):
$if N + 8 < NR:
- int8x16_t vout${M}x${ABC[N:N+16]} = vmovn_high_s16(vmovn_s16(vacc${M}x${ABC[N:N+8]}), vacc${M}x${ABC[N+8:N+16]});
+ int8x16_t vout${M}x${ABC[N:N+16]} = vuzp1q_s8(vreinterpretq_s8_s16(vacc${M}x${ABC[N:N+8]}), vreinterpretq_s8_s16(vacc${M}x${ABC[N+8:N+16]}));
$elif M % 2 == 1:
- int8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vmovn_high_s16(vmovn_s16(vacc${M-1}x${ABC[N:N+8]}), vacc${M}x${ABC[N:N+8]});
+ int8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vuzp1q_s8(vreinterpretq_s8_s16(vacc${M-1}x${ABC[N:N+8]}), vreinterpretq_s8_s16(vacc${M}x${ABC[N:N+8]}));
$elif M + 1 == MR:
int8x8_t vout${M}x${ABC[N:N+8]} = vmovn_s16(vacc${M}x${ABC[N:N+8]});
$else:
diff --git a/src/qs8-igemm/c8-neon-mull-padal.c.in b/src/qs8-igemm/c8-neon-mull-padal.c.in
index 9dbc412..d0dcffa 100644
--- a/src/qs8-igemm/c8-neon-mull-padal.c.in
+++ b/src/qs8-igemm/c8-neon-mull-padal.c.in
@@ -192,14 +192,14 @@
$if REQUANTIZATION == "FP32" and not ARMV8:
$for M in range(MR):
$for N in range(0, NR, 8):
- const int16x8_t vacc${M}x${ABC[N:N+8]} = vmovn_high_s32(vmovn_s32(vacc${M}x${ABC[N:N+4]}), vacc${M}x${ABC[N+4:N+8]});
+ const int16x8_t vacc${M}x${ABC[N:N+8]} = vuzp1q_s16(vreinterpretq_s16_s32(vacc${M}x${ABC[N:N+4]}), vreinterpretq_s16_s32(vacc${M}x${ABC[N+4:N+8]}));
$for M in range(MR):
$for N in range(0, NR, 16):
$if N + 8 < NR:
- int8x16_t vout${M}x${ABC[N:N+16]} = vmovn_high_s16(vmovn_s16(vacc${M}x${ABC[N:N+8]}), vacc${M}x${ABC[N+8:N+16]});
+ int8x16_t vout${M}x${ABC[N:N+16]} = vuzp1q_s8(vreinterpretq_s8_s16(vacc${M}x${ABC[N:N+8]}), vreinterpretq_s8_s16(vacc${M}x${ABC[N+8:N+16]}));
$elif M % 2 == 1:
- int8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vmovn_high_s16(vmovn_s16(vacc${M-1}x${ABC[N:N+8]}), vacc${M}x${ABC[N:N+8]});
+ int8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vuzp1q_s8(vreinterpretq_s8_s16(vacc${M-1}x${ABC[N:N+8]}), vreinterpretq_s8_s16(vacc${M}x${ABC[N:N+8]}));
$elif M + 1 == MR:
int8x8_t vout${M}x${ABC[N:N+8]} = vmovn_s16(vacc${M}x${ABC[N:N+8]});
$else:
diff --git a/src/qs8-igemm/gen/1x16-minmax-fp32-neon-mlal-lane.c b/src/qs8-igemm/gen/1x16-minmax-fp32-neon-mlal-lane.c
index 38b07f4..b67dfaa 100644
--- a/src/qs8-igemm/gen/1x16-minmax-fp32-neon-mlal-lane.c
+++ b/src/qs8-igemm/gen/1x16-minmax-fp32-neon-mlal-lane.c
@@ -271,10 +271,10 @@
vacc0xCDEF = vsubq_s32(vacc0xCDEF, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc0x01234567 = vmovn_high_s32(vmovn_s32(vacc0x0123), vacc0x4567);
- const int16x8_t vacc0x89ABCDEF = vmovn_high_s32(vmovn_s32(vacc0x89AB), vacc0xCDEF);
+ const int16x8_t vacc0x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0x0123), vreinterpretq_s16_s32(vacc0x4567));
+ const int16x8_t vacc0x89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc0x89AB), vreinterpretq_s16_s32(vacc0xCDEF));
- int8x16_t vout0x0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc0x01234567), vacc0x89ABCDEF);
+ int8x16_t vout0x0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc0x01234567), vreinterpretq_s8_s16(vacc0x89ABCDEF));
#else
const int16x8_t vacc0x01234567 = vcombine_s16(vmovn_s32(vacc0x0123), vmovn_s32(vacc0x4567));
const int16x8_t vacc0x89ABCDEF = vcombine_s16(vmovn_s32(vacc0x89AB), vmovn_s32(vacc0xCDEF));
diff --git a/src/qs8-igemm/gen/1x8c8-minmax-fp32-neon-mlal-padal.c b/src/qs8-igemm/gen/1x8c8-minmax-fp32-neon-mlal-padal.c
index 602d752..7631809 100644
--- a/src/qs8-igemm/gen/1x8c8-minmax-fp32-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/1x8c8-minmax-fp32-neon-mlal-padal.c
@@ -195,7 +195,7 @@
vacc0x4567 = vsubq_s32(vacc0x4567, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc0x01234567 = vmovn_high_s32(vmovn_s32(vacc0x0123), vacc0x4567);
+ const int16x8_t vacc0x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0x0123), vreinterpretq_s16_s32(vacc0x4567));
int8x8_t vout0x01234567 = vmovn_s16(vacc0x01234567);
#else
diff --git a/src/qs8-igemm/gen/2x8c8-minmax-fp32-neon-mlal-padal.c b/src/qs8-igemm/gen/2x8c8-minmax-fp32-neon-mlal-padal.c
index 26fbc33..e58565e 100644
--- a/src/qs8-igemm/gen/2x8c8-minmax-fp32-neon-mlal-padal.c
+++ b/src/qs8-igemm/gen/2x8c8-minmax-fp32-neon-mlal-padal.c
@@ -286,10 +286,10 @@
vacc1x4567 = vsubq_s32(vacc1x4567, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc0x01234567 = vmovn_high_s32(vmovn_s32(vacc0x0123), vacc0x4567);
- const int16x8_t vacc1x01234567 = vmovn_high_s32(vmovn_s32(vacc1x0123), vacc1x4567);
+ const int16x8_t vacc0x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0x0123), vreinterpretq_s16_s32(vacc0x4567));
+ const int16x8_t vacc1x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc1x0123), vreinterpretq_s16_s32(vacc1x4567));
- int8x16_t vout0x01234567_1x01234567 = vmovn_high_s16(vmovn_s16(vacc0x01234567), vacc1x01234567);
+ int8x16_t vout0x01234567_1x01234567 = vuzp1q_s8(vreinterpretq_s8_s16(vacc0x01234567), vreinterpretq_s8_s16(vacc1x01234567));
#else
const int16x8_t vacc0x01234567 = vcombine_s16(vmovn_s32(vacc0x0123), vmovn_s32(vacc0x4567));
const int16x8_t vacc1x01234567 = vcombine_s16(vmovn_s32(vacc1x0123), vmovn_s32(vacc1x4567));
diff --git a/src/qs8-igemm/gen/4x16-minmax-fp32-neon-mlal-lane.c b/src/qs8-igemm/gen/4x16-minmax-fp32-neon-mlal-lane.c
index b1b79d4..0369d6b 100644
--- a/src/qs8-igemm/gen/4x16-minmax-fp32-neon-mlal-lane.c
+++ b/src/qs8-igemm/gen/4x16-minmax-fp32-neon-mlal-lane.c
@@ -571,19 +571,19 @@
vacc3xCDEF = vsubq_s32(vacc3xCDEF, vmagic_bias_less_zero_point);
#if XNN_ARCH_ARM64
- const int16x8_t vacc0x01234567 = vmovn_high_s32(vmovn_s32(vacc0x0123), vacc0x4567);
- const int16x8_t vacc0x89ABCDEF = vmovn_high_s32(vmovn_s32(vacc0x89AB), vacc0xCDEF);
- const int16x8_t vacc1x01234567 = vmovn_high_s32(vmovn_s32(vacc1x0123), vacc1x4567);
- const int16x8_t vacc1x89ABCDEF = vmovn_high_s32(vmovn_s32(vacc1x89AB), vacc1xCDEF);
- const int16x8_t vacc2x01234567 = vmovn_high_s32(vmovn_s32(vacc2x0123), vacc2x4567);
- const int16x8_t vacc2x89ABCDEF = vmovn_high_s32(vmovn_s32(vacc2x89AB), vacc2xCDEF);
- const int16x8_t vacc3x01234567 = vmovn_high_s32(vmovn_s32(vacc3x0123), vacc3x4567);
- const int16x8_t vacc3x89ABCDEF = vmovn_high_s32(vmovn_s32(vacc3x89AB), vacc3xCDEF);
+ const int16x8_t vacc0x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0x0123), vreinterpretq_s16_s32(vacc0x4567));
+ const int16x8_t vacc0x89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc0x89AB), vreinterpretq_s16_s32(vacc0xCDEF));
+ const int16x8_t vacc1x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc1x0123), vreinterpretq_s16_s32(vacc1x4567));
+ const int16x8_t vacc1x89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc1x89AB), vreinterpretq_s16_s32(vacc1xCDEF));
+ const int16x8_t vacc2x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc2x0123), vreinterpretq_s16_s32(vacc2x4567));
+ const int16x8_t vacc2x89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc2x89AB), vreinterpretq_s16_s32(vacc2xCDEF));
+ const int16x8_t vacc3x01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc3x0123), vreinterpretq_s16_s32(vacc3x4567));
+ const int16x8_t vacc3x89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc3x89AB), vreinterpretq_s16_s32(vacc3xCDEF));
- int8x16_t vout0x0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc0x01234567), vacc0x89ABCDEF);
- int8x16_t vout1x0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc1x01234567), vacc1x89ABCDEF);
- int8x16_t vout2x0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc2x01234567), vacc2x89ABCDEF);
- int8x16_t vout3x0123456789ABCDEF = vmovn_high_s16(vmovn_s16(vacc3x01234567), vacc3x89ABCDEF);
+ int8x16_t vout0x0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc0x01234567), vreinterpretq_s8_s16(vacc0x89ABCDEF));
+ int8x16_t vout1x0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc1x01234567), vreinterpretq_s8_s16(vacc1x89ABCDEF));
+ int8x16_t vout2x0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc2x01234567), vreinterpretq_s8_s16(vacc2x89ABCDEF));
+ int8x16_t vout3x0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc3x01234567), vreinterpretq_s8_s16(vacc3x89ABCDEF));
#else
const int16x8_t vacc0x01234567 = vcombine_s16(vmovn_s32(vacc0x0123), vmovn_s32(vacc0x4567));
const int16x8_t vacc0x89ABCDEF = vcombine_s16(vmovn_s32(vacc0x89AB), vmovn_s32(vacc0xCDEF));
diff --git a/src/qs8-igemm/neon-mlal-lane.c.in b/src/qs8-igemm/neon-mlal-lane.c.in
index 6e9c968..7c0eb94 100644
--- a/src/qs8-igemm/neon-mlal-lane.c.in
+++ b/src/qs8-igemm/neon-mlal-lane.c.in
@@ -243,14 +243,14 @@
$if REQUANTIZATION == "FP32" and not ARMV8:
$for M in range(MR):
$for N in range(0, NR, 8):
- const int16x8_t vacc${M}x${ABC[N:N+8]} = vmovn_high_s32(vmovn_s32(vacc${M}x${ABC[N:N+4]}), vacc${M}x${ABC[N+4:N+8]});
+ const int16x8_t vacc${M}x${ABC[N:N+8]} = vuzp1q_s16(vreinterpretq_s16_s32(vacc${M}x${ABC[N:N+4]}), vreinterpretq_s16_s32(vacc${M}x${ABC[N+4:N+8]}));
$for M in range(MR):
$for N in range(0, NR, 16):
$if N + 8 < NR:
- int8x16_t vout${M}x${ABC[N:N+16]} = vmovn_high_s16(vmovn_s16(vacc${M}x${ABC[N:N+8]}), vacc${M}x${ABC[N+8:N+16]});
+ int8x16_t vout${M}x${ABC[N:N+16]} = vuzp1q_s8(vreinterpretq_s8_s16(vacc${M}x${ABC[N:N+8]}), vreinterpretq_s8_s16(vacc${M}x${ABC[N+8:N+16]}));
$elif M % 2 == 1:
- int8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vmovn_high_s16(vmovn_s16(vacc${M-1}x${ABC[N:N+8]}), vacc${M}x${ABC[N:N+8]});
+ int8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vuzp1q_s8(vreinterpretq_s8_s16(vacc${M-1}x${ABC[N:N+8]}), vreinterpretq_s8_s16(vacc${M}x${ABC[N:N+8]}));
$elif M + 1 == MR:
int8x8_t vout${M}x${ABC[N:N+8]} = vmovn_s16(vacc${M}x${ABC[N:N+8]});
$else: