Use specialized layouts in SSE4/AVX2 QS8 [I]GEMM & DWCONV microkernels
PiperOrigin-RevId: 375310512
diff --git a/src/qs8-igemm/MRx16c8-avx512skx.c.in b/src/qs8-igemm/MRx16c8-avx512skx.c.in
index 55595b7..4eddff2 100644
--- a/src/qs8-igemm/MRx16c8-avx512skx.c.in
+++ b/src/qs8-igemm/MRx16c8-avx512skx.c.in
@@ -64,13 +64,18 @@
const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
$if MR > 1:
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
$else:
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ $if MR > 2:
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ $elif MR == 2:
+ const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_min));
+ const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ $else:
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
$for N in range(4, 16, 4):
@@ -155,26 +160,28 @@
_mm512_mask_sub_epi32(vacc${M}x084C195D2A6E3B7F, _mm512_cmpgt_epi32_mask(vrem${M}x084C195D2A6E3B7F, vremainder_threshold), vacc${M}x084C195D2A6E3B7F, vminus_one);
$if MR == 1:
- __m256i vacc0x084C2A6E195D3B7F = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc0x084C195D2A6E3B7F), _mm512_extracti32x8_epi32(vacc0x084C195D2A6E3B7F, 1)), voutput_zero_point);
- vacc0x084C2A6E195D3B7F = _mm256_min_epi16(_mm256_max_epi16(vacc0x084C2A6E195D3B7F, voutput_min), voutput_max);
+ const __m256i vacc0x084C2A6E195D3B7F = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc0x084C195D2A6E3B7F), _mm512_extracti32x8_epi32(vacc0x084C195D2A6E3B7F, 1)), voutput_zero_point);
$else:
$for M in range(0, MR, 2):
- __m512i vacc${M}${min(M+1, MR-1)}x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc${M}x084C195D2A6E3B7F, vacc${min(M+1, MR-1)}x084C195D2A6E3B7F), voutput_zero_point);
-
- $for M in range(0, MR, 2):
- vacc${M}${min(M+1, MR-1)}x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc${M}${min(M+1, MR-1)}x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
+ const __m512i vacc${M}${min(M+1, MR-1)}x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc${M}x084C195D2A6E3B7F, vacc${min(M+1, MR-1)}x084C195D2A6E3B7F), voutput_zero_point);
$if MR > 2:
__m512i vout012${min(3, MR-1)}x084Cx195Dx2A6Ex3B7F = _mm512_packs_epi16(vacc01x084Cx195Dx2A6Ex3B7F, vacc2${min(3, MR-1)}x084Cx195Dx2A6Ex3B7F);
vout012${min(M+3, MR-1)}x084Cx195Dx2A6Ex3B7F = _mm512_permutexvar_epi32(_mm512_set_epi32(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0), vout012${min(3, MR-1)}x084Cx195Dx2A6Ex3B7F);
- const __m512i vout012${min(3, MR-1)}x0123456789ABCDEF = _mm512_shuffle_epi8(vout012${min(3, MR-1)}x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ __m512i vout012${min(3, MR-1)}x0123456789ABCDEF = _mm512_shuffle_epi8(vout012${min(3, MR-1)}x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ vout012${min(3, MR-1)}x0123456789ABCDEF = _mm512_max_epi8(vout012${min(3, MR-1)}x0123456789ABCDEF, voutput_min);
+ vout012${min(3, MR-1)}x0123456789ABCDEF = _mm512_min_epi8(vout012${min(3, MR-1)}x0123456789ABCDEF, voutput_max);
$elif MR == 2:
const __m256i vout01x084Cx2A6Ex195Dx3B7F = _mm256_packs_epi16(_mm512_castsi512_si256(vacc01x084Cx195Dx2A6Ex3B7F), _mm512_extracti32x8_epi32(vacc01x084Cx195Dx2A6Ex3B7F, 1));
const __m256i vout01x084C2A6E195D3B7F = _mm256_permutevar8x32_epi32(vout01x084Cx2A6Ex195Dx3B7F, _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0));
- const __m256i vout01x0123456789ABCDEF = _mm256_shuffle_epi8(vout01x084C2A6E195D3B7F, _mm256_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0, 15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ __m256i vout01x0123456789ABCDEF = _mm256_shuffle_epi8(vout01x084C2A6E195D3B7F, _mm256_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0, 15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ vout01x0123456789ABCDEF = _mm256_max_epi8(vout01x0123456789ABCDEF, voutput_min);
+ vout01x0123456789ABCDEF = _mm256_min_epi8(vout01x0123456789ABCDEF, voutput_max);
$elif MR == 1:
const __m128i vout0x084C2A6E195D3B7F = _mm_packs_epi16(_mm256_castsi256_si128(vacc0x084C2A6E195D3B7F), _mm256_extracti128_si256(vacc0x084C2A6E195D3B7F, 1));
- const __m128i vout0x0123456789ABCDEF = _mm_shuffle_epi8(vout0x084C2A6E195D3B7F, _mm_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ __m128i vout0x0123456789ABCDEF = _mm_shuffle_epi8(vout0x084C2A6E195D3B7F, _mm_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ vout0x0123456789ABCDEF = _mm_max_epi8(vout0x0123456789ABCDEF, voutput_min);
+ vout0x0123456789ABCDEF = _mm_min_epi8(vout0x0123456789ABCDEF, voutput_max);
$if MR > 2:
if (nc >= 16) {
diff --git a/src/qs8-igemm/MRx4c2-sse.c.in b/src/qs8-igemm/MRx4c2-sse.c.in
index e3cc0df..ded0cde 100644
--- a/src/qs8-igemm/MRx4c2-sse.c.in
+++ b/src/qs8-igemm/MRx4c2-sse.c.in
@@ -25,6 +25,7 @@
#include <xnnpack/math.h>
+$PARAMS_STRUCT = "sse4" if SSE >= 4 else "sse2"
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
void xnn_qs8_igemm_minmax_ukernel_${MR}x4c2__${ISA}_${VARIANT.lower()}(
size_t mr,
@@ -192,8 +193,8 @@
p -= ${MR} * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
$if SSE == 4:
$for M in range(MR):
@@ -255,31 +256,39 @@
$for M in range(MR):
const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
$for M in range(MR):
const __m128i vrem${M}x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
$for M in range(MR):
vacc${M}x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
$for M in range(0, MR, 2):
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- $for M in range(0, MR, 2):
- vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
+ $if SSE < 4:
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ $for M in range(0, MR, 2):
+ vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
$if MR > 2:
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc${min(2, MR-1)}${min(3, MR-1)}x0123);
$else:
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
+ $if SSE == 4:
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
$for M in reversed(range(1, MR)):
$if SSE == 4:
diff --git a/src/qs8-igemm/MRx4c8-sse.c.in b/src/qs8-igemm/MRx4c8-sse.c.in
index 0b4b236..2357c61 100644
--- a/src/qs8-igemm/MRx4c8-sse.c.in
+++ b/src/qs8-igemm/MRx4c8-sse.c.in
@@ -25,6 +25,7 @@
#include <xnnpack/math.h>
+$PARAMS_STRUCT = "sse4" if SSE >= 4 else "sse2"
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
void xnn_qs8_igemm_minmax_ukernel_${MR}x4c8__${ISA}_${VARIANT.lower()}(
size_t mr,
@@ -150,8 +151,8 @@
$for M in range(MR):
__m128i vacc${M}x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc${M}x02, vacc${M}x13), _mm_unpackhi_epi32(vacc${M}x02, vacc${M}x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
$if SSE == 4:
$for M in range(MR):
@@ -213,31 +214,39 @@
$for M in range(MR):
const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
$for M in range(MR):
const __m128i vrem${M}x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
$for M in range(MR):
vacc${M}x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
$for M in range(0, MR, 2):
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- $for M in range(0, MR, 2):
- vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
+ $if SSE < 4:
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ $for M in range(0, MR, 2):
+ vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
$if MR > 2:
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc${min(2, MR-1)}${min(3, MR-1)}x0123);
$else:
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
+ $if SSE == 4:
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
$for M in reversed(range(1, MR)):
$if SSE == 4:
diff --git a/src/qs8-igemm/MRx8c8-avx2.c.in b/src/qs8-igemm/MRx8c8-avx2.c.in
index 582f70c..34df8bb 100644
--- a/src/qs8-igemm/MRx8c8-avx2.c.in
+++ b/src/qs8-igemm/MRx8c8-avx2.c.in
@@ -111,8 +111,8 @@
$for M in range(MR):
__m256i vacc${M}x01234567 = _mm256_permutevar8x32_epi32(vacc${M}x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
$for M in range(MR):
const __m256i vacc${M}x11335577 = _mm256_shuffle_epi32(vacc${M}x01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -130,33 +130,35 @@
$for M in range(MR):
const __m256i vq31prod${M}x01234567 = _mm256_blend_epi16(vq31prod${M}x0246, vq31prod${M}x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
$for M in range(MR):
const __m256i vrem${M}x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod${M}x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${M}x01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
$for M in range(MR):
vacc${M}x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod${M}x01234567, vshift), _mm256_cmpgt_epi32(vrem${M}x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
$for M in range(0, MR, 2):
__m256i vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc${M}x01234567, vacc${min(M+1, MR-1)}x01234567), voutput_zero_point);
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_permute4x64_epi64(vacc${M}${min(M+1, MR-1)}x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- $for M in range(0, MR, 2):
- vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc${M}${min(M+1, MR-1)}x01234567, voutput_min), voutput_max);
-
$if MR > 2:
__m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc${min(2, MR-1)}${min(3, MR-1)}x01234567);
$else:
__m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-igemm/gen/1x16c8-minmax-avx512skx.c b/src/qs8-igemm/gen/1x16c8-minmax-avx512skx.c
index 6e860c2..fd29a93 100644
--- a/src/qs8-igemm/gen/1x16c8-minmax-avx512skx.c
+++ b/src/qs8-igemm/gen/1x16c8-minmax-avx512skx.c
@@ -49,9 +49,9 @@
const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
@@ -116,11 +116,12 @@
vacc0x084C195D2A6E3B7F =
_mm512_mask_sub_epi32(vacc0x084C195D2A6E3B7F, _mm512_cmpgt_epi32_mask(vrem0x084C195D2A6E3B7F, vremainder_threshold), vacc0x084C195D2A6E3B7F, vminus_one);
- __m256i vacc0x084C2A6E195D3B7F = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc0x084C195D2A6E3B7F), _mm512_extracti32x8_epi32(vacc0x084C195D2A6E3B7F, 1)), voutput_zero_point);
- vacc0x084C2A6E195D3B7F = _mm256_min_epi16(_mm256_max_epi16(vacc0x084C2A6E195D3B7F, voutput_min), voutput_max);
+ const __m256i vacc0x084C2A6E195D3B7F = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc0x084C195D2A6E3B7F), _mm512_extracti32x8_epi32(vacc0x084C195D2A6E3B7F, 1)), voutput_zero_point);
const __m128i vout0x084C2A6E195D3B7F = _mm_packs_epi16(_mm256_castsi256_si128(vacc0x084C2A6E195D3B7F), _mm256_extracti128_si256(vacc0x084C2A6E195D3B7F, 1));
- const __m128i vout0x0123456789ABCDEF = _mm_shuffle_epi8(vout0x084C2A6E195D3B7F, _mm_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ __m128i vout0x0123456789ABCDEF = _mm_shuffle_epi8(vout0x084C2A6E195D3B7F, _mm_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ vout0x0123456789ABCDEF = _mm_max_epi8(vout0x0123456789ABCDEF, voutput_min);
+ vout0x0123456789ABCDEF = _mm_min_epi8(vout0x0123456789ABCDEF, voutput_max);
if (nc >= 16) {
_mm_storeu_si128((__m128i*) c0, vout0x0123456789ABCDEF);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-avx-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-avx-ld128.c
index 2443e03..f309d27 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-avx-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-avx-ld128.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,24 +132,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-avx-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-avx-ld64.c
index b846272..852be0e 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-avx-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-avx-ld64.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,24 +132,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld128.c
index 8bcb718..74c720c 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld128.c
@@ -166,6 +166,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld64.c
index 8909f0d..12f9ea0 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld64.c
@@ -166,6 +166,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld128.c
index dc9dbeb..f3a9c3a 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld128.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,24 +132,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld64.c
index cc8372c..4e9e5f9 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld64.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,24 +132,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld128.c
index 5f22e7c..4380fab 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld128.c
@@ -166,6 +166,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld64.c
index 7c923f1..655c345 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld64.c
@@ -166,6 +166,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-xop-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-xop-ld128.c
index 37cbccf..95a2775 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-xop-ld128.c
@@ -123,8 +123,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -137,24 +137,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-xop-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-xop-ld64.c
index 81f421d..841e214 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-xop-ld64.c
@@ -123,8 +123,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -137,24 +137,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-avx-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-avx-ld128.c
index 2d2d988..58d01b6 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-avx-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-avx-ld128.c
@@ -90,8 +90,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -104,24 +104,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-avx-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-avx-ld64.c
index 9283765..d2dd638 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-avx-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-avx-ld64.c
@@ -92,8 +92,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -106,24 +106,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld128.c
index 1b225a1..1793705 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld128.c
@@ -138,6 +138,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld64.c
index 0700e32..5d1fdae 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld64.c
@@ -140,6 +140,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld128.c
index 0ba0ed2..eaffaa6 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld128.c
@@ -90,8 +90,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -104,24 +104,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld64.c
index 5f96910..ad2344f 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld64.c
@@ -92,8 +92,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -106,24 +106,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld128.c
index 0c0454f..da9a1c1 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld128.c
@@ -138,6 +138,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld64.c
index 5de63b6..520ba62 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld64.c
@@ -140,6 +140,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-xop-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-xop-ld128.c
index 1c95c42..b0646d5 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-xop-ld128.c
@@ -95,8 +95,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -109,24 +109,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-xop-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-xop-ld64.c
index aa6bbd1..7403cf2 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-xop-ld64.c
@@ -97,8 +97,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -111,24 +111,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x8c8-minmax-avx2.c b/src/qs8-igemm/gen/1x8c8-minmax-avx2.c
index 4372a66..b988ae6 100644
--- a/src/qs8-igemm/gen/1x8c8-minmax-avx2.c
+++ b/src/qs8-igemm/gen/1x8c8-minmax-avx2.c
@@ -104,8 +104,8 @@
const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc0x11335577 = _mm256_shuffle_epi32(vacc0x01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -118,25 +118,25 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vacc00x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc0x01234567), voutput_zero_point);
vacc00x01234567 = _mm256_permute4x64_epi64(vacc00x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vacc00x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc00x01234567, voutput_min), voutput_max);
-
__m256i vout = _mm256_packs_epi16(vacc00x01234567, vacc00x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-igemm/gen/2x16c8-minmax-avx512skx.c b/src/qs8-igemm/gen/2x16c8-minmax-avx512skx.c
index af495ea..1a3b682 100644
--- a/src/qs8-igemm/gen/2x16c8-minmax-avx512skx.c
+++ b/src/qs8-igemm/gen/2x16c8-minmax-avx512skx.c
@@ -53,9 +53,9 @@
const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_min));
+ const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
@@ -148,13 +148,13 @@
vacc1x084C195D2A6E3B7F =
_mm512_mask_sub_epi32(vacc1x084C195D2A6E3B7F, _mm512_cmpgt_epi32_mask(vrem1x084C195D2A6E3B7F, vremainder_threshold), vacc1x084C195D2A6E3B7F, vminus_one);
- __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
-
- vacc01x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc01x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
+ const __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
const __m256i vout01x084Cx2A6Ex195Dx3B7F = _mm256_packs_epi16(_mm512_castsi512_si256(vacc01x084Cx195Dx2A6Ex3B7F), _mm512_extracti32x8_epi32(vacc01x084Cx195Dx2A6Ex3B7F, 1));
const __m256i vout01x084C2A6E195D3B7F = _mm256_permutevar8x32_epi32(vout01x084Cx2A6Ex195Dx3B7F, _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0));
- const __m256i vout01x0123456789ABCDEF = _mm256_shuffle_epi8(vout01x084C2A6E195D3B7F, _mm256_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0, 15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ __m256i vout01x0123456789ABCDEF = _mm256_shuffle_epi8(vout01x084C2A6E195D3B7F, _mm256_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0, 15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ vout01x0123456789ABCDEF = _mm256_max_epi8(vout01x0123456789ABCDEF, voutput_min);
+ vout01x0123456789ABCDEF = _mm256_min_epi8(vout01x0123456789ABCDEF, voutput_max);
if (nc >= 16) {
_mm_storeu_si128((__m128i*) c1, _mm256_extracti128_si256(vout01x0123456789ABCDEF, 1));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-avx-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-avx-ld128.c
index 6c2f64a..cc08b92 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-avx-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-avx-ld128.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -167,28 +167,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-avx-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-avx-ld64.c
index 92c3ee4..21823a3 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-avx-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-avx-ld64.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -167,28 +167,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld128.c
index 2ae8eba..89f3698 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld128.c
@@ -213,6 +213,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld64.c
index c1ccebd..d59bf28 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld64.c
@@ -213,6 +213,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld128.c
index 10a96a7..b852014 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld128.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -167,28 +167,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld64.c
index b50953e..4532c35 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld64.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -167,28 +167,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld128.c
index 0b72444..10fc1e4 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld128.c
@@ -213,6 +213,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld64.c
index 29c6aeb..175955b 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld64.c
@@ -213,6 +213,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-xop-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-xop-ld128.c
index 4cbf968..71b9cab 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-xop-ld128.c
@@ -152,8 +152,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -172,28 +172,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-xop-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-xop-ld64.c
index 2f53629..88649df 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-xop-ld64.c
@@ -152,8 +152,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -172,28 +172,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-avx-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-avx-ld128.c
index 05fed74..9307dc8 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-avx-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-avx-ld128.c
@@ -112,8 +112,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,28 +132,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-avx-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-avx-ld64.c
index 774b901..b92f417 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-avx-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-avx-ld64.c
@@ -114,8 +114,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -134,28 +134,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld128.c
index 6444d15..9304588 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld128.c
@@ -178,6 +178,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld64.c
index 5f948d5..c6773a8 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld64.c
@@ -180,6 +180,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld128.c
index daaacf7..0c1f2ab 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld128.c
@@ -112,8 +112,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,28 +132,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld64.c
index 4ad6652..96af574 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld64.c
@@ -114,8 +114,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -134,28 +134,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld128.c
index 99bb004..95a3f59 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld128.c
@@ -178,6 +178,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld64.c
index 971df2a..ec11e48 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld64.c
@@ -180,6 +180,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-xop-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-xop-ld128.c
index ed3cc5c..bd3c5a7 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-xop-ld128.c
@@ -117,8 +117,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -137,28 +137,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-xop-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-xop-ld64.c
index 2032a31..bb0a7c2 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-xop-ld64.c
@@ -119,8 +119,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -139,28 +139,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x8c8-minmax-avx2.c b/src/qs8-igemm/gen/2x8c8-minmax-avx2.c
index a82458a..b33bfa9 100644
--- a/src/qs8-igemm/gen/2x8c8-minmax-avx2.c
+++ b/src/qs8-igemm/gen/2x8c8-minmax-avx2.c
@@ -127,8 +127,8 @@
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc0x11335577 = _mm256_shuffle_epi32(vacc0x01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc1x11335577 = _mm256_shuffle_epi32(vacc1x01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -147,29 +147,29 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod1x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod1x01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod1x01234567, vshift), _mm256_cmpgt_epi32(vrem1x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vacc01x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc01x01234567, voutput_min), voutput_max);
-
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc01x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-igemm/gen/3x16c8-minmax-avx512skx.c b/src/qs8-igemm/gen/3x16c8-minmax-avx512skx.c
index 722d4f3..73d32f0 100644
--- a/src/qs8-igemm/gen/3x16c8-minmax-avx512skx.c
+++ b/src/qs8-igemm/gen/3x16c8-minmax-avx512skx.c
@@ -57,9 +57,9 @@
const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
@@ -180,15 +180,14 @@
vacc2x084C195D2A6E3B7F =
_mm512_mask_sub_epi32(vacc2x084C195D2A6E3B7F, _mm512_cmpgt_epi32_mask(vrem2x084C195D2A6E3B7F, vremainder_threshold), vacc2x084C195D2A6E3B7F, vminus_one);
- __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
- __m512i vacc22x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc2x084C195D2A6E3B7F, vacc2x084C195D2A6E3B7F), voutput_zero_point);
-
- vacc01x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc01x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
- vacc22x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc22x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
+ const __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
+ const __m512i vacc22x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc2x084C195D2A6E3B7F, vacc2x084C195D2A6E3B7F), voutput_zero_point);
__m512i vout0122x084Cx195Dx2A6Ex3B7F = _mm512_packs_epi16(vacc01x084Cx195Dx2A6Ex3B7F, vacc22x084Cx195Dx2A6Ex3B7F);
vout0122x084Cx195Dx2A6Ex3B7F = _mm512_permutexvar_epi32(_mm512_set_epi32(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0), vout0122x084Cx195Dx2A6Ex3B7F);
- const __m512i vout0122x0123456789ABCDEF = _mm512_shuffle_epi8(vout0122x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ __m512i vout0122x0123456789ABCDEF = _mm512_shuffle_epi8(vout0122x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ vout0122x0123456789ABCDEF = _mm512_max_epi8(vout0122x0123456789ABCDEF, voutput_min);
+ vout0122x0123456789ABCDEF = _mm512_min_epi8(vout0122x0123456789ABCDEF, voutput_max);
if (nc >= 16) {
_mm_storeu_si128((__m128i*) c2, _mm512_extracti32x4_epi32(vout0122x0123456789ABCDEF, 2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-avx-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-avx-ld128.c
index 94a8a7c..07170b4 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-avx-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-avx-ld128.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -202,7 +202,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -210,8 +210,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -219,17 +219,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-avx-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-avx-ld64.c
index 2f1a22d..a6224db 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-avx-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-avx-ld64.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -202,7 +202,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -210,8 +210,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -219,17 +219,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld128.c
index 5933dd2..991e596 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld128.c
@@ -243,7 +243,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -262,6 +262,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld64.c
index 0e380ee..1dff2c6 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld64.c
@@ -243,7 +243,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -262,6 +262,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld128.c
index b441fae..e9ac047 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld128.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -202,7 +202,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -210,8 +210,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -219,17 +219,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld64.c
index 6223c65..cf64636 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld64.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -202,7 +202,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -210,8 +210,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -219,17 +219,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld128.c
index 0b2a7bc..fedb5a0 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld128.c
@@ -243,7 +243,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -262,6 +262,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld64.c
index ab0c741..0a105fb 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld64.c
@@ -243,7 +243,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -262,6 +262,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-xop-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-xop-ld128.c
index 1ffbfee..6fd187b 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-xop-ld128.c
@@ -181,8 +181,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -207,7 +207,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -215,8 +215,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -224,17 +224,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-xop-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-xop-ld64.c
index 30a0e81..c0a46f4 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-xop-ld64.c
@@ -181,8 +181,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -207,7 +207,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -215,8 +215,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -224,17 +224,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-avx-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-avx-ld128.c
index ee09dd0..7d5fff1 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-avx-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-avx-ld128.c
@@ -134,8 +134,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -160,7 +160,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -168,8 +168,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -177,17 +177,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-avx-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-avx-ld64.c
index dae0701..330eaa0 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-avx-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-avx-ld64.c
@@ -136,8 +136,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -162,7 +162,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -170,8 +170,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -179,17 +179,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld128.c
index c0d9941..0022e00 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld128.c
@@ -201,7 +201,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -220,6 +220,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld64.c
index aaf3649..83d4334 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld64.c
@@ -203,7 +203,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -222,6 +222,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld128.c
index 92c755b..a39c3ab 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld128.c
@@ -134,8 +134,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -160,7 +160,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -168,8 +168,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -177,17 +177,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld64.c
index 069b8aa..82eec9e 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld64.c
@@ -136,8 +136,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -162,7 +162,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -170,8 +170,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -179,17 +179,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld128.c
index 69adc9b..0a35711 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld128.c
@@ -201,7 +201,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -220,6 +220,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld64.c
index 6afce4a..27e040c 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld64.c
@@ -203,7 +203,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -222,6 +222,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-xop-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-xop-ld128.c
index 0ffb9cc..96d42c0 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-xop-ld128.c
@@ -139,8 +139,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -165,7 +165,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -173,8 +173,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -182,17 +182,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-xop-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-xop-ld64.c
index b7d59b6..f311e3c 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-xop-ld64.c
@@ -141,8 +141,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -167,7 +167,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -175,8 +175,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -184,17 +184,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x8c8-minmax-avx2.c b/src/qs8-igemm/gen/3x8c8-minmax-avx2.c
index 202bb77..2055125 100644
--- a/src/qs8-igemm/gen/3x8c8-minmax-avx2.c
+++ b/src/qs8-igemm/gen/3x8c8-minmax-avx2.c
@@ -150,8 +150,8 @@
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
__m256i vacc2x01234567 = _mm256_permutevar8x32_epi32(vacc2x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc0x11335577 = _mm256_shuffle_epi32(vacc0x01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc1x11335577 = _mm256_shuffle_epi32(vacc1x01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -176,7 +176,7 @@
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
const __m256i vq31prod2x01234567 = _mm256_blend_epi16(vq31prod2x0246, vq31prod2x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
@@ -184,8 +184,8 @@
const __m256i vrem2x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod2x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod2x01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
@@ -193,19 +193,18 @@
vacc2x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod2x01234567, vshift), _mm256_cmpgt_epi32(vrem2x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
__m256i vacc22x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc2x01234567, vacc2x01234567), voutput_zero_point);
vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
vacc22x01234567 = _mm256_permute4x64_epi64(vacc22x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vacc01x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc01x01234567, voutput_min), voutput_max);
- vacc22x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc22x01234567, voutput_min), voutput_max);
-
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc22x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-igemm/gen/4x16c8-minmax-avx512skx.c b/src/qs8-igemm/gen/4x16c8-minmax-avx512skx.c
index 4828185..2752f79 100644
--- a/src/qs8-igemm/gen/4x16c8-minmax-avx512skx.c
+++ b/src/qs8-igemm/gen/4x16c8-minmax-avx512skx.c
@@ -61,9 +61,9 @@
const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
@@ -212,15 +212,14 @@
vacc3x084C195D2A6E3B7F =
_mm512_mask_sub_epi32(vacc3x084C195D2A6E3B7F, _mm512_cmpgt_epi32_mask(vrem3x084C195D2A6E3B7F, vremainder_threshold), vacc3x084C195D2A6E3B7F, vminus_one);
- __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
- __m512i vacc23x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc2x084C195D2A6E3B7F, vacc3x084C195D2A6E3B7F), voutput_zero_point);
-
- vacc01x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc01x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
- vacc23x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc23x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
+ const __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
+ const __m512i vacc23x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc2x084C195D2A6E3B7F, vacc3x084C195D2A6E3B7F), voutput_zero_point);
__m512i vout0123x084Cx195Dx2A6Ex3B7F = _mm512_packs_epi16(vacc01x084Cx195Dx2A6Ex3B7F, vacc23x084Cx195Dx2A6Ex3B7F);
vout0123x084Cx195Dx2A6Ex3B7F = _mm512_permutexvar_epi32(_mm512_set_epi32(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0), vout0123x084Cx195Dx2A6Ex3B7F);
- const __m512i vout0123x0123456789ABCDEF = _mm512_shuffle_epi8(vout0123x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ __m512i vout0123x0123456789ABCDEF = _mm512_shuffle_epi8(vout0123x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ vout0123x0123456789ABCDEF = _mm512_max_epi8(vout0123x0123456789ABCDEF, voutput_min);
+ vout0123x0123456789ABCDEF = _mm512_min_epi8(vout0123x0123456789ABCDEF, voutput_max);
if (nc >= 16) {
_mm_storeu_si128((__m128i*) c3, _mm512_extracti32x4_epi32(vout0123x0123456789ABCDEF, 3));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-avx-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-avx-ld128.c
index 90e9f37..e341603 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-avx-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-avx-ld128.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -237,7 +237,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -247,8 +247,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -258,17 +258,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-avx-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-avx-ld64.c
index 64aa971..2d815c9 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-avx-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-avx-ld64.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -237,7 +237,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -247,8 +247,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -258,17 +258,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld128.c
index 725a7aa..3e86a97 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld128.c
@@ -288,7 +288,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -309,6 +309,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld64.c
index 636f501..bdb9caa 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld64.c
@@ -288,7 +288,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -309,6 +309,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld128.c
index 0d511f1..09e76c0 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld128.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -237,7 +237,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -247,8 +247,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -258,17 +258,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld64.c
index 894e46a..66c499c 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld64.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -237,7 +237,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -247,8 +247,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -258,17 +258,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld128.c
index 2d1aba8..d8559fe 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld128.c
@@ -288,7 +288,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -309,6 +309,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld64.c
index 11ff0c6..2318f10 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld64.c
@@ -288,7 +288,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -309,6 +309,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-xop-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-xop-ld128.c
index 16a0c0a..0ce5964 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-xop-ld128.c
@@ -210,8 +210,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -242,7 +242,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -252,8 +252,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -263,17 +263,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-xop-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-xop-ld64.c
index b4b0e7e..b44fa57 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-xop-ld64.c
@@ -210,8 +210,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -242,7 +242,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -252,8 +252,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -263,17 +263,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);