Use specialized layouts in SSE4/AVX2 QS8 [I]GEMM & DWCONV microkernels
PiperOrigin-RevId: 375310512
diff --git a/src/init.c b/src/init.c
index 489772f..5aa36d9 100644
--- a/src/init.c
+++ b/src/init.c
@@ -1560,7 +1560,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_4x16c8__avx512skx);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x16c8__avx512skx);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x16c8__avx512skx);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_gemm_sse2_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_gemm_sse4_params;
xnn_params.qs8.gemm.mr = 4;
xnn_params.qs8.gemm.nr = 16;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -1570,7 +1570,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_2x4c8__xop_ld64);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x4c8__xop_ld64);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x4c8__xop_ld64);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_gemm_sse2_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_gemm_sse4_params;
xnn_params.qs8.gemm.mr = 2;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -1579,7 +1579,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_3x8c8__avx2);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x8c8__avx2);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x8c8__avx2);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_gemm_sse2_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_gemm_avx2_params;
xnn_params.qs8.gemm.mr = 3;
xnn_params.qs8.gemm.nr = 8;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -1588,7 +1588,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_2x4c8__avx_ld128);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x4c8__avx_ld128);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x4c8__avx_ld128);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_gemm_sse2_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_gemm_sse4_params;
xnn_params.qs8.gemm.mr = 2;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -1597,7 +1597,7 @@
xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_3x4c8__sse41_ld64);
xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_ukernel_1x4c8__sse41_ld64);
xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_ukernel_1x4c8__sse41_ld64);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_gemm_sse2_params;
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_gemm_sse4_params;
xnn_params.qs8.gemm.mr = 3;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -1631,31 +1631,31 @@
} else if (cpuinfo_has_x86_xop()) {
// XOP should be checked before AVX2: AMD Excavator supports both, but performs better with XOP microkernels
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up16x9__xop_mul32;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_gemm_sse2_params;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_gemm_sse4_params;
xnn_params.qs8.dwconv[0].channel_tile = 16;
xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up16x25__xop_mul32;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_gemm_sse2_params;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_gemm_sse4_params;
xnn_params.qs8.dwconv[1].channel_tile = 16;
} else if (cpuinfo_has_x86_avx2()) {
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up16x9__avx2_mul32;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_gemm_sse2_params;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_gemm_avx2_params;
xnn_params.qs8.dwconv[0].channel_tile = 16;
xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up16x25__avx2_mul32;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_gemm_sse2_params;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_gemm_avx2_params;
xnn_params.qs8.dwconv[1].channel_tile = 16;
} else if (cpuinfo_has_x86_avx()) {
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up16x9__avx_mul32;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_gemm_sse2_params;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_gemm_sse4_params;
xnn_params.qs8.dwconv[0].channel_tile = 16;
xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up16x25__avx_mul32;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_gemm_sse2_params;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_gemm_sse4_params;
xnn_params.qs8.dwconv[1].channel_tile = 16;
} else if (cpuinfo_has_x86_sse4_1()) {
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up8x9__sse41_mul16;
- xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_gemm_sse2_params;
+ xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_gemm_sse4_params;
xnn_params.qs8.dwconv[0].channel_tile = 8;
xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up8x25__sse41_mul16;
- xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_gemm_sse2_params;
+ xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_gemm_sse4_params;
xnn_params.qs8.dwconv[1].channel_tile = 8;
} else if (cpuinfo_has_x86_ssse3()) {
xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_ukernel_up8x9__ssse3_mul16;
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-avx-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-avx-mul16.c
index fd78bf2..201e81c 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-avx-mul16.c
@@ -693,8 +693,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -723,7 +723,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -733,8 +733,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -744,17 +744,19 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
}
@@ -1093,8 +1095,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -1113,28 +1115,28 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-avx-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-avx-mul32.c
index 58e7828..715c5ab 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-avx-mul32.c
@@ -544,8 +544,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -574,7 +574,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -584,8 +584,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -595,16 +595,15 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
@@ -768,8 +767,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -781,23 +780,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-avx2-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-avx2-mul16.c
index 74b833a..15c168f 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-avx2-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-avx2-mul16.c
@@ -391,8 +391,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -410,28 +410,29 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
-
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vout012389AB4567CDEF = _mm256_min_epi16(_mm256_max_epi16(vout012389AB4567CDEF, voutput_min), voutput_max);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
}
@@ -642,8 +643,8 @@
vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod24x89ABCDEF));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -661,29 +662,28 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_min_epi8(_mm_max_epi8(vout0123456789ABCDEF, voutput_min), voutput_max);
if (c & 8) {
_mm_storel_epi64((__m128i*) output, vout0123456789ABCDEF);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-avx2-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-avx2-mul32.c
index 47c0ad6..37d3bcc 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-avx2-mul32.c
@@ -391,8 +391,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -410,28 +410,29 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vout012389AB4567CDEF = _mm256_min_epi16(_mm256_max_epi16(vout012389AB4567CDEF, voutput_min), voutput_max);
-
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
}
@@ -594,8 +595,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -607,23 +608,23 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-sse2-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-sse2-mul16.c
index b9a5594..13a5160 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-sse2-mul16.c
@@ -771,7 +771,7 @@
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -786,12 +786,16 @@
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
+ vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
+
const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
+ vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
+ vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
}
@@ -1177,7 +1181,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -1186,12 +1190,12 @@
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-sse41-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-sse41-mul16.c
index 528e594..16037c0 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-sse41-mul16.c
@@ -693,8 +693,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -723,7 +723,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -733,8 +733,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -744,17 +744,19 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
}
@@ -1093,8 +1095,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -1113,28 +1115,28 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-sse41-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-sse41-mul32.c
index 5d0962f..f332747 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-sse41-mul32.c
@@ -544,8 +544,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -574,7 +574,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -584,8 +584,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -595,16 +595,15 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
@@ -768,8 +767,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -781,23 +780,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-ssse3-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-ssse3-mul16.c
index 1b7af91..938d0e5 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-ssse3-mul16.c
@@ -771,7 +771,7 @@
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -786,12 +786,16 @@
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
+ vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
+
const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
+ vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
+ vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
}
@@ -1177,7 +1181,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -1186,12 +1190,12 @@
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-xop-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-xop-mul32.c
index 41f7791..9dc6211 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-xop-mul32.c
@@ -549,8 +549,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 400 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -579,7 +579,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -589,8 +589,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -600,16 +600,15 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
@@ -773,8 +772,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -786,23 +785,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-avx-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-avx-mul16.c
index c125f1f..a8bb56a 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-avx-mul16.c
@@ -277,8 +277,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -307,7 +307,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -317,8 +317,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -328,17 +328,19 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
}
@@ -469,8 +471,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -489,28 +491,28 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-avx-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-avx-mul32.c
index 19dae96..45c2fa8 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-avx-mul32.c
@@ -224,8 +224,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -254,7 +254,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -264,8 +264,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -275,16 +275,15 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
@@ -352,8 +351,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -365,23 +364,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-avx2-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-avx2-mul16.c
index 1dc427b..b1db9e6 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-avx2-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-avx2-mul16.c
@@ -167,8 +167,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -186,28 +186,29 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
-
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vout012389AB4567CDEF = _mm256_min_epi16(_mm256_max_epi16(vout012389AB4567CDEF, voutput_min), voutput_max);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
}
@@ -290,8 +291,8 @@
vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod8x89ABCDEF));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -309,29 +310,28 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_min_epi8(_mm_max_epi8(vout0123456789ABCDEF, voutput_min), voutput_max);
if (c & 8) {
_mm_storel_epi64((__m128i*) output, vout0123456789ABCDEF);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-avx2-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-avx2-mul32.c
index c23fc81..b03ec06 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-avx2-mul32.c
@@ -167,8 +167,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -186,28 +186,29 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vout012389AB4567CDEF = _mm256_min_epi16(_mm256_max_epi16(vout012389AB4567CDEF, voutput_min), voutput_max);
-
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
}
@@ -274,8 +275,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -287,23 +288,23 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-sse2-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-sse2-mul16.c
index ee07325..96df3cd 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-sse2-mul16.c
@@ -355,7 +355,7 @@
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -370,12 +370,16 @@
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
+ vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
+
const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
+ vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
+ vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
}
@@ -553,7 +557,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -562,12 +566,12 @@
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-sse41-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-sse41-mul16.c
index 3e2f7f8..eadc573 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-sse41-mul16.c
@@ -277,8 +277,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -307,7 +307,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -317,8 +317,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -328,17 +328,19 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
}
@@ -469,8 +471,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -489,28 +491,28 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-sse41-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-sse41-mul32.c
index 9cd4988..580cfb2 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-sse41-mul32.c
@@ -224,8 +224,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -254,7 +254,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -264,8 +264,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -275,16 +275,15 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
@@ -352,8 +351,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -365,23 +364,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-ssse3-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-ssse3-mul16.c
index 90f98d8..3d148fe 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-ssse3-mul16.c
@@ -355,7 +355,7 @@
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -370,12 +370,16 @@
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
+ vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
+
const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
+ vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
+ vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
}
@@ -553,7 +557,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -562,12 +566,12 @@
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-xop-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-xop-mul32.c
index 9a1b0e1..d1590b2 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-xop-mul32.c
@@ -229,8 +229,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 144 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -259,7 +259,7 @@
const __m128i vq31prod89AB = _mm_blend_epi16(vq31prod8A, vq31prod9B, 0xCC);
const __m128i vq31prodCDEF = _mm_blend_epi16(vq31prodCE, vq31prodDF, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -269,8 +269,8 @@
const __m128i vremCDEF =
_mm_add_epi32(_mm_and_si128(vq31prodCDEF, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodCDEF));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -280,16 +280,15 @@
vaccCDEF =
_mm_sub_epi32(_mm_sra_epi32(vq31prodCDEF, vshift), _mm_cmpgt_epi32(vremCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
output += 16;
@@ -357,8 +356,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -370,23 +369,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-avx-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-avx-mul16.c
index 63b6dc2..5040e72 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-avx-mul16.c
@@ -895,8 +895,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 600 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -937,7 +937,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -951,8 +951,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -966,20 +966,23 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, voutput_min), voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
+
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
output += 24;
@@ -1319,8 +1322,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -1339,28 +1342,28 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-avx-mul32.c b/src/qs8-dwconv/gen/up24x25-minmax-avx-mul32.c
index 4d40b5c..f32c5d5 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-avx-mul32.c
@@ -696,8 +696,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 600 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -738,7 +738,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -752,8 +752,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -767,19 +767,19 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+ voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
+ voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
@@ -944,8 +944,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -957,23 +957,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-avx2-mul32.c b/src/qs8-dwconv/gen/up24x25-minmax-avx2-mul32.c
index d37b9aa..30157d0 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-avx2-mul32.c
@@ -467,8 +467,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 600 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -492,7 +492,7 @@
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
const __m256i vq31prodGHIJKLMN = _mm256_blend_epi16(vq31prodGIKM, vq31prodHJLN, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
@@ -500,8 +500,8 @@
const __m256i vremGHIJKLMN =
_mm256_add_epi32(_mm256_and_si256(vq31prodGHIJKLMN, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prodGHIJKLMN));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
@@ -509,18 +509,20 @@
vaccGHIJKLMN =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prodGHIJKLMN, vshift), _mm256_cmpgt_epi32(vremGHIJKLMN, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vaccGHIJKLMN), _mm256_extracti128_si256(vaccGHIJKLMN, 1)), _mm256_castsi256_si128(voutput_zero_point));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vout012389AB4567CDEF = _mm256_min_epi16(_mm256_max_epi16(vout012389AB4567CDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, _mm256_castsi256_si128(voutput_min)), _mm256_castsi256_si128(voutput_max));
-
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
+ voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
output += 24;
@@ -684,8 +686,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -697,23 +699,23 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-sse2-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-sse2-mul16.c
index c58b6a0..8ba83d2 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-sse2-mul16.c
@@ -1005,7 +1005,7 @@
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -1025,14 +1025,19 @@
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
+ vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
+ voutGHIJKLMN = _mm_max_epi16(voutGHIJKLMN, voutput_min);
+
const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, voutput_min), voutput_max);
+ vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
+ vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
+ voutGHIJKLMN = _mm_min_epi16(voutGHIJKLMN, voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
output += 24;
@@ -1419,7 +1424,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -1428,12 +1433,12 @@
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-sse41-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-sse41-mul16.c
index 5cc20c6..d0069bb 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-sse41-mul16.c
@@ -895,8 +895,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 600 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -937,7 +937,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -951,8 +951,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -966,20 +966,23 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, voutput_min), voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
+
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
output += 24;
@@ -1319,8 +1322,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -1339,28 +1342,28 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-sse41-mul32.c b/src/qs8-dwconv/gen/up24x25-minmax-sse41-mul32.c
index 4ad2e4f..3309700 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-sse41-mul32.c
@@ -696,8 +696,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 600 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -738,7 +738,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -752,8 +752,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -767,19 +767,19 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+ voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
+ voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
@@ -944,8 +944,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -957,23 +957,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-ssse3-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-ssse3-mul16.c
index 358b3c3..499ab88 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-ssse3-mul16.c
@@ -1005,7 +1005,7 @@
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -1025,14 +1025,19 @@
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
+ vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
+ voutGHIJKLMN = _mm_max_epi16(voutGHIJKLMN, voutput_min);
+
const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, voutput_min), voutput_max);
+ vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
+ vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
+ voutGHIJKLMN = _mm_min_epi16(voutGHIJKLMN, voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
output += 24;
@@ -1419,7 +1424,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -1428,12 +1433,12 @@
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-xop-mul32.c b/src/qs8-dwconv/gen/up24x25-minmax-xop-mul32.c
index 91838ed..fae92d2 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-xop-mul32.c
@@ -701,8 +701,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 600 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -743,7 +743,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -757,8 +757,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -772,19 +772,19 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+ voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
+ voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
@@ -949,8 +949,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -962,23 +962,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-avx-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-avx-mul16.c
index ad996c3..1582948 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-avx-mul16.c
@@ -351,8 +351,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -393,7 +393,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -407,8 +407,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -422,20 +422,23 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, voutput_min), voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
+
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
output += 24;
@@ -567,8 +570,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -587,28 +590,28 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-avx-mul32.c b/src/qs8-dwconv/gen/up24x9-minmax-avx-mul32.c
index 8569133..32bb9dd 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-avx-mul32.c
@@ -280,8 +280,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -322,7 +322,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -336,8 +336,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -351,19 +351,19 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+ voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
+ voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
@@ -432,8 +432,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -445,23 +445,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-avx2-mul32.c b/src/qs8-dwconv/gen/up24x9-minmax-avx2-mul32.c
index f04663f..26221c8 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-avx2-mul32.c
@@ -195,8 +195,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -220,7 +220,7 @@
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
const __m256i vq31prodGHIJKLMN = _mm256_blend_epi16(vq31prodGIKM, vq31prodHJLN, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
@@ -228,8 +228,8 @@
const __m256i vremGHIJKLMN =
_mm256_add_epi32(_mm256_and_si256(vq31prodGHIJKLMN, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prodGHIJKLMN));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
@@ -237,18 +237,20 @@
vaccGHIJKLMN =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prodGHIJKLMN, vshift), _mm256_cmpgt_epi32(vremGHIJKLMN, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vaccGHIJKLMN), _mm256_extracti128_si256(vaccGHIJKLMN, 1)), _mm256_castsi256_si128(voutput_zero_point));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vout012389AB4567CDEF = _mm256_min_epi16(_mm256_max_epi16(vout012389AB4567CDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, _mm256_castsi256_si128(voutput_min)), _mm256_castsi256_si128(voutput_max));
-
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
+ voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
output += 24;
@@ -316,8 +318,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -329,23 +331,23 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-sse2-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-sse2-mul16.c
index 8eda6f8..c2a24ed 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-sse2-mul16.c
@@ -461,7 +461,7 @@
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -481,14 +481,19 @@
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
+ vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
+ voutGHIJKLMN = _mm_max_epi16(voutGHIJKLMN, voutput_min);
+
const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, voutput_min), voutput_max);
+ vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
+ vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
+ voutGHIJKLMN = _mm_min_epi16(voutGHIJKLMN, voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
output += 24;
@@ -667,7 +672,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -676,12 +681,12 @@
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-sse41-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-sse41-mul16.c
index 29b11f3..b681da4 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-sse41-mul16.c
@@ -351,8 +351,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -393,7 +393,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -407,8 +407,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -422,20 +422,23 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, voutput_min), voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
+
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
output += 24;
@@ -567,8 +570,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -587,28 +590,28 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-sse41-mul32.c b/src/qs8-dwconv/gen/up24x9-minmax-sse41-mul32.c
index c587239..65decb9 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-sse41-mul32.c
@@ -280,8 +280,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -322,7 +322,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -336,8 +336,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -351,19 +351,19 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+ voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
+ voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
@@ -432,8 +432,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -445,23 +445,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-ssse3-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-ssse3-mul16.c
index d3934a0..46cd248 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-ssse3-mul16.c
@@ -461,7 +461,7 @@
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -481,14 +481,19 @@
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
+ vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
+ voutGHIJKLMN = _mm_max_epi16(voutGHIJKLMN, voutput_min);
+
const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, voutput_min), voutput_max);
+ vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
+ vout89ABCDEF = _mm_min_epi16(vout89ABCDEF, voutput_max);
+ voutGHIJKLMN = _mm_min_epi16(voutGHIJKLMN, voutput_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
output += 24;
@@ -667,7 +672,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -676,12 +681,12 @@
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-xop-mul32.c b/src/qs8-dwconv/gen/up24x9-minmax-xop-mul32.c
index 6924e5c..9aa0bda 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-xop-mul32.c
@@ -285,8 +285,8 @@
w = (const void*) ((uintptr_t) w + 24 * sizeof(int32_t) + 216 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -327,7 +327,7 @@
const __m128i vq31prodGHIJ = _mm_blend_epi16(vq31prodGI, vq31prodHJ, 0xCC);
const __m128i vq31prodKLMN = _mm_blend_epi16(vq31prodKM, vq31prodLN, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
@@ -341,8 +341,8 @@
const __m128i vremKLMN =
_mm_add_epi32(_mm_and_si128(vq31prodKLMN, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prodKLMN));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -356,19 +356,19 @@
vaccKLMN =
_mm_sub_epi32(_mm_sra_epi32(vq31prodKLMN, vshift), _mm_cmpgt_epi32(vremKLMN, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
__m128i voutGHIJKLMN = _mm_adds_epi16(_mm_packs_epi32(vaccGHIJ, vaccKLMN), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
- voutGHIJKLMN = _mm_min_epi16(_mm_max_epi16(voutGHIJKLMN, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
__m128i voutGHIJKLMNGHIJKLMN = _mm_packs_epi16(voutGHIJKLMN, voutGHIJKLMN);
+ voutGHIJKLMNGHIJKLMN = _mm_max_epi8(voutGHIJKLMNGHIJKLMN, voutput_min);
+ voutGHIJKLMNGHIJKLMN = _mm_min_epi8(voutGHIJKLMNGHIJKLMN, voutput_max);
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storel_epi64((__m128i*) (output + 16), voutGHIJKLMNGHIJKLMN);
@@ -437,8 +437,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -450,23 +450,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up32x25-minmax-avx2-mul16.c b/src/qs8-dwconv/gen/up32x25-minmax-avx2-mul16.c
index a0452f0..e22d41f 100644
--- a/src/qs8-dwconv/gen/up32x25-minmax-avx2-mul16.c
+++ b/src/qs8-dwconv/gen/up32x25-minmax-avx2-mul16.c
@@ -543,8 +543,8 @@
w = (const void*) ((uintptr_t) w + 32 * sizeof(int32_t) + 800 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -574,7 +574,7 @@
const __m256i vq31prodGHIJKLMN = _mm256_blend_epi16(vq31prodGIKM, vq31prodHJLN, 0xCC);
const __m256i vq31prodOPQRSTUV = _mm256_blend_epi16(vq31prodOQSU, vq31prodPRTV, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
@@ -584,8 +584,8 @@
const __m256i vremOPQRSTUV =
_mm256_add_epi32(_mm256_and_si256(vq31prodOPQRSTUV, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prodOPQRSTUV));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
@@ -595,18 +595,20 @@
vaccOPQRSTUV =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prodOPQRSTUV, vshift), _mm256_cmpgt_epi32(vremOPQRSTUV, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
- __m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
-
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vout012389AB4567CDEF = _mm256_min_epi16(_mm256_max_epi16(vout012389AB4567CDEF, voutput_min), voutput_max);
- voutGHIJOPQRKLMNSTUV = _mm256_min_epi16(_mm256_max_epi16(voutGHIJOPQRKLMNSTUV, voutput_min), voutput_max);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
+ const __m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
__m128i voutGHIJKLMNOPQRSTUV = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(voutGHIJOPQRKLMNSTUV), _mm256_extracti128_si256(voutGHIJOPQRKLMNSTUV, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNOPQRSTUV = _mm_max_epi8(voutGHIJKLMNOPQRSTUV, voutput_min);
+ voutGHIJKLMNOPQRSTUV = _mm_min_epi8(voutGHIJKLMNOPQRSTUV, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storeu_si128((__m128i*) (output + 16), voutGHIJKLMNOPQRSTUV);
output += 32;
@@ -846,8 +848,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t));
k += 16;
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -865,29 +867,28 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_min_epi8(_mm_max_epi8(vout0123456789ABCDEF, voutput_min), voutput_max);
if XNN_LIKELY(c >= 16) {
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
diff --git a/src/qs8-dwconv/gen/up32x25-minmax-avx2-mul32.c b/src/qs8-dwconv/gen/up32x25-minmax-avx2-mul32.c
index 8218e16..268ac2a 100644
--- a/src/qs8-dwconv/gen/up32x25-minmax-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up32x25-minmax-avx2-mul32.c
@@ -543,8 +543,8 @@
w = (const void*) ((uintptr_t) w + 32 * sizeof(int32_t) + 800 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -574,7 +574,7 @@
const __m256i vq31prodGHIJKLMN = _mm256_blend_epi16(vq31prodGIKM, vq31prodHJLN, 0xCC);
const __m256i vq31prodOPQRSTUV = _mm256_blend_epi16(vq31prodOQSU, vq31prodPRTV, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
@@ -584,8 +584,8 @@
const __m256i vremOPQRSTUV =
_mm256_add_epi32(_mm256_and_si256(vq31prodOPQRSTUV, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prodOPQRSTUV));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
@@ -595,18 +595,20 @@
vaccOPQRSTUV =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prodOPQRSTUV, vshift), _mm256_cmpgt_epi32(vremOPQRSTUV, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
__m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vout012389AB4567CDEF = _mm256_min_epi16(_mm256_max_epi16(vout012389AB4567CDEF, voutput_min), voutput_max);
- voutGHIJOPQRKLMNSTUV = _mm256_min_epi16(_mm256_max_epi16(voutGHIJOPQRKLMNSTUV, voutput_min), voutput_max);
-
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
__m128i voutGHIJKLMNOPQRSTUV = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(voutGHIJOPQRKLMNSTUV), _mm256_extracti128_si256(voutGHIJOPQRKLMNSTUV, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNOPQRSTUV = _mm_max_epi8(voutGHIJKLMNOPQRSTUV, voutput_min);
+ voutGHIJKLMNOPQRSTUV = _mm_min_epi8(voutGHIJKLMNOPQRSTUV, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storeu_si128((__m128i*) (output + 16), voutGHIJKLMNOPQRSTUV);
output += 32;
@@ -770,8 +772,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -783,23 +785,23 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up32x9-minmax-avx2-mul16.c b/src/qs8-dwconv/gen/up32x9-minmax-avx2-mul16.c
index 4972c11..a7be837 100644
--- a/src/qs8-dwconv/gen/up32x9-minmax-avx2-mul16.c
+++ b/src/qs8-dwconv/gen/up32x9-minmax-avx2-mul16.c
@@ -223,8 +223,8 @@
w = (const void*) ((uintptr_t) w + 32 * sizeof(int32_t) + 288 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -254,7 +254,7 @@
const __m256i vq31prodGHIJKLMN = _mm256_blend_epi16(vq31prodGIKM, vq31prodHJLN, 0xCC);
const __m256i vq31prodOPQRSTUV = _mm256_blend_epi16(vq31prodOQSU, vq31prodPRTV, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
@@ -264,8 +264,8 @@
const __m256i vremOPQRSTUV =
_mm256_add_epi32(_mm256_and_si256(vq31prodOPQRSTUV, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prodOPQRSTUV));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
@@ -275,18 +275,20 @@
vaccOPQRSTUV =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prodOPQRSTUV, vshift), _mm256_cmpgt_epi32(vremOPQRSTUV, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
- __m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
-
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vout012389AB4567CDEF = _mm256_min_epi16(_mm256_max_epi16(vout012389AB4567CDEF, voutput_min), voutput_max);
- voutGHIJOPQRKLMNSTUV = _mm256_min_epi16(_mm256_max_epi16(voutGHIJOPQRKLMNSTUV, voutput_min), voutput_max);
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
+ const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
+ const __m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
__m128i voutGHIJKLMNOPQRSTUV = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(voutGHIJOPQRKLMNSTUV), _mm256_extracti128_si256(voutGHIJOPQRKLMNSTUV, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNOPQRSTUV = _mm_max_epi8(voutGHIJKLMNOPQRSTUV, voutput_min);
+ voutGHIJKLMNOPQRSTUV = _mm_min_epi8(voutGHIJKLMNOPQRSTUV, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storeu_si128((__m128i*) (output + 16), voutGHIJKLMNOPQRSTUV);
output += 32;
@@ -382,8 +384,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t));
k += 16;
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -401,29 +403,28 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
const __m256i vq31prod89ABCDEF = _mm256_blend_epi16(vq31prod8ACE, vq31prod9BDF, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
_mm256_add_epi32(_mm256_and_si256(vq31prod89ABCDEF, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod89ABCDEF));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod89ABCDEF, vshift), _mm256_cmpgt_epi32(vrem89ABCDEF, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
__m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
- vout89ABCDEF = _mm_min_epi16(_mm_max_epi16(vout89ABCDEF, voutput_min), voutput_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
__m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
+ vout0123456789ABCDEF = _mm_min_epi8(_mm_max_epi8(vout0123456789ABCDEF, voutput_min), voutput_max);
if XNN_LIKELY(c >= 16) {
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
diff --git a/src/qs8-dwconv/gen/up32x9-minmax-avx2-mul32.c b/src/qs8-dwconv/gen/up32x9-minmax-avx2-mul32.c
index 579b30b..1e2a93e 100644
--- a/src/qs8-dwconv/gen/up32x9-minmax-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up32x9-minmax-avx2-mul32.c
@@ -223,8 +223,8 @@
w = (const void*) ((uintptr_t) w + 32 * sizeof(int32_t) + 288 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc9BDF = _mm256_shuffle_epi32(vacc89ABCDEF, _MM_SHUFFLE(3, 3, 1, 1));
@@ -254,7 +254,7 @@
const __m256i vq31prodGHIJKLMN = _mm256_blend_epi16(vq31prodGIKM, vq31prodHJLN, 0xCC);
const __m256i vq31prodOPQRSTUV = _mm256_blend_epi16(vq31prodOQSU, vq31prodPRTV, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
const __m256i vrem89ABCDEF =
@@ -264,8 +264,8 @@
const __m256i vremOPQRSTUV =
_mm256_add_epi32(_mm256_and_si256(vq31prodOPQRSTUV, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prodOPQRSTUV));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
vacc89ABCDEF =
@@ -275,18 +275,20 @@
vaccOPQRSTUV =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prodOPQRSTUV, vshift), _mm256_cmpgt_epi32(vremOPQRSTUV, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
__m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vout012389AB4567CDEF = _mm256_min_epi16(_mm256_max_epi16(vout012389AB4567CDEF, voutput_min), voutput_max);
- voutGHIJOPQRKLMNSTUV = _mm256_min_epi16(_mm256_max_epi16(voutGHIJOPQRKLMNSTUV, voutput_min), voutput_max);
-
__m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
__m128i voutGHIJKLMNOPQRSTUV = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(voutGHIJOPQRKLMNSTUV), _mm256_extracti128_si256(voutGHIJOPQRKLMNSTUV, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
+ vout0123456789ABCDEF = _mm_min_epi8(vout0123456789ABCDEF, voutput_max);
+ voutGHIJKLMNOPQRSTUV = _mm_max_epi8(voutGHIJKLMNOPQRSTUV, voutput_min);
+ voutGHIJKLMNOPQRSTUV = _mm_min_epi8(voutGHIJKLMNOPQRSTUV, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
_mm_storeu_si128((__m128i*) (output + 16), voutGHIJKLMNOPQRSTUV);
output += 32;
@@ -354,8 +356,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -367,23 +369,23 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-avx-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-avx-mul16.c
index 0ab2a49..b23e417 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-avx-mul16.c
@@ -491,8 +491,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 200 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -509,28 +509,31 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
}
@@ -841,8 +844,8 @@
vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vp24x01234567lo, vp24x01234567hi));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -861,28 +864,28 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (c & 4) {
*((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-avx-mul32.c b/src/qs8-dwconv/gen/up8x25-minmax-avx-mul32.c
index ae3ec16..889b7a8 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-avx-mul32.c
@@ -392,8 +392,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 200 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -410,27 +410,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
@@ -594,8 +594,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -607,23 +607,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-avx2-mul32.c b/src/qs8-dwconv/gen/up8x25-minmax-avx2-mul32.c
index cdabcdf..a298d77 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-avx2-mul32.c
@@ -315,8 +315,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 200 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -328,24 +328,25 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
}
@@ -480,8 +481,8 @@
vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi24x01234567, vk24x01234567));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -493,23 +494,23 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
if (c & 4) {
*((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-sse2-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-sse2-mul16.c
index 493b0bb..498844c 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-sse2-mul16.c
@@ -537,7 +537,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -547,11 +547,14 @@
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
+
const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
}
@@ -909,7 +912,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -918,12 +921,12 @@
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
if (c & 4) {
*((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-sse41-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-sse41-mul16.c
index da1a284..45dfe82 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-sse41-mul16.c
@@ -491,8 +491,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 200 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -509,28 +509,31 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
}
@@ -841,8 +844,8 @@
vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vp24x01234567lo, vp24x01234567hi));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -861,28 +864,28 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (c & 4) {
*((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-sse41-mul32.c b/src/qs8-dwconv/gen/up8x25-minmax-sse41-mul32.c
index ce92c24..3ccbf5a 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-sse41-mul32.c
@@ -392,8 +392,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 200 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -410,27 +410,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
@@ -594,8 +594,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -607,23 +607,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-ssse3-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-ssse3-mul16.c
index 35f720d..5704b3b 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-ssse3-mul16.c
@@ -537,7 +537,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -547,11 +547,14 @@
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
+
const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
}
@@ -909,7 +912,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -918,12 +921,12 @@
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
if (c & 4) {
*((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-xop-mul32.c b/src/qs8-dwconv/gen/up8x25-minmax-xop-mul32.c
index 657ece3..a4467b8 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-xop-mul32.c
@@ -397,8 +397,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 200 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -415,27 +415,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
@@ -599,8 +599,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -612,23 +612,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-avx-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-avx-mul16.c
index 0b9ee4f..7f4f9e7 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-avx-mul16.c
@@ -203,8 +203,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -221,28 +221,31 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
}
@@ -361,8 +364,8 @@
vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vp8x01234567lo, vp8x01234567hi));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -381,28 +384,28 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (c & 4) {
*((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-avx-mul32.c b/src/qs8-dwconv/gen/up8x9-minmax-avx-mul32.c
index 11db437..20ae162 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-avx-mul32.c
@@ -168,8 +168,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -186,27 +186,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
@@ -274,8 +274,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -287,23 +287,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-avx2-mul32.c b/src/qs8-dwconv/gen/up8x9-minmax-avx2-mul32.c
index 3e82ebf..fdebd15 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-avx2-mul32.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-avx2-mul32.c
@@ -139,8 +139,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -152,24 +152,25 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
}
@@ -224,8 +225,8 @@
vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc1357 = _mm256_shuffle_epi32(vacc01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -237,23 +238,23 @@
const __m256i vq31prod01234567 = _mm256_blend_epi16(vq31prod0246, vq31prod1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod01234567, vshift), _mm256_cmpgt_epi32(vrem01234567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
if (c & 4) {
*((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-sse2-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-sse2-mul16.c
index 554a00a..d9bb1c1 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-sse2-mul16.c
@@ -249,7 +249,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -259,11 +259,14 @@
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
+
const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
}
@@ -429,7 +432,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -438,12 +441,12 @@
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
if (c & 4) {
*((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-sse41-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-sse41-mul16.c
index 38d707a..0a422aa 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-sse41-mul16.c
@@ -203,8 +203,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -221,28 +221,31 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
+
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
}
@@ -361,8 +364,8 @@
vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vp8x01234567lo, vp8x01234567hi));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc57 = _mm_shuffle_epi32(vacc4567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -381,28 +384,28 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (c & 4) {
*((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-sse41-mul32.c b/src/qs8-dwconv/gen/up8x9-minmax-sse41-mul32.c
index 1ca6748..4cb309e 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-sse41-mul32.c
@@ -168,8 +168,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -186,27 +186,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
@@ -274,8 +274,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -287,23 +287,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-ssse3-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-ssse3-mul16.c
index f21866a..b1f1651 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-ssse3-mul16.c
@@ -249,7 +249,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -259,11 +259,14 @@
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
+
const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_min_epi16(vout01234567, voutput_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
}
@@ -429,7 +432,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
@@ -438,12 +441,12 @@
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
+ vout01234567 = _mm_max_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_min));
+ vout01234567 = _mm_min_epi16(vout01234567, _mm_load_si128((const __m128i*) params->sse2.output_max));
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+
if (c & 4) {
*((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout0123456701234567);
vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-xop-mul32.c b/src/qs8-dwconv/gen/up8x9-minmax-xop-mul32.c
index cb711f2..f3d8308 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-xop-mul32.c
@@ -173,8 +173,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vprod02 = _mm_add_epi64(_mm_mul_epi32(vacc0123, vmultiplier), vrounding);
@@ -191,27 +191,27 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
const __m128i vq31prod4567 = _mm_blend_epi16(vq31prod46, vq31prod57, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
const __m128i vrem4567 =
_mm_add_epi32(_mm_and_si128(vq31prod4567, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod4567));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout01234567 = _mm_min_epi16(_mm_max_epi16(vout01234567, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
__m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
+ vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
+ vout0123456701234567 = _mm_min_epi8(vout0123456701234567, voutput_max);
_mm_storel_epi64((__m128i*) output, vout0123456701234567);
output += 8;
@@ -279,8 +279,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc13 = _mm_shuffle_epi32(vacc0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -292,23 +292,21 @@
const __m128i vq31prod0123 = _mm_blend_epi16(vq31prod02, vq31prod13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout0123 = _mm_min_epi16(_mm_max_epi16(vout0123, voutput_min), voutput_max);
-
vout0123 = _mm_packs_epi16(vout0123, vout0123);
+ vout0123 = _mm_max_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout0123 = _mm_min_epi8(vout0123, _mm_load_si128((const __m128i*) params->sse4.output_max));
if XNN_LIKELY(c >= 4) {
_mm_storeu_si32(output, vout0123);
diff --git a/src/qs8-dwconv/unipass-avx2-mul16.c.in b/src/qs8-dwconv/unipass-avx2-mul16.c.in
index fb2be5e..f61650f 100644
--- a/src/qs8-dwconv/unipass-avx2-mul16.c.in
+++ b/src/qs8-dwconv/unipass-avx2-mul16.c.in
@@ -63,8 +63,8 @@
w = (const void*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(int32_t) + ${KERNEL_TILE * CHANNEL_TILE} * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
$for C in range(0, CHANNEL_TILE, 8):
const __m256i vacc${ABC[C+1:C+8:2]} = _mm256_shuffle_epi32(vacc${ABC[C:C+8]}, _MM_SHUFFLE(3, 3, 1, 1));
@@ -80,29 +80,33 @@
$for C in range(0, CHANNEL_TILE, 8):
const __m256i vq31prod${ABC[C:C+8]} = _mm256_blend_epi16(vq31prod${ABC[C:C+8:2]}, vq31prod${ABC[C+1:C+8:2]}, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
$for C in range(0, CHANNEL_TILE, 8):
const __m256i vrem${ABC[C:C+8]} =
_mm256_add_epi32(_mm256_and_si256(vq31prod${ABC[C:C+8]}, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${ABC[C:C+8]}));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ $if CHANNEL_TILE > 8:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
$for C in range(0, CHANNEL_TILE, 8):
vacc${ABC[C:C+8]} =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod${ABC[C:C+8]}, vshift), _mm256_cmpgt_epi32(vrem${ABC[C:C+8]}, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
$for C in range(0, CHANNEL_TILE, 16):
- __m256i vout${ABC[C:C+4]}${ABC[C+8:C+12]}${ABC[C+4:C+8]}${ABC[C+12:C+16]} = _mm256_adds_epi16(_mm256_packs_epi32(vacc${ABC[C:C+8]}, vacc${ABC[C+8:C+16]}), voutput_zero_point);
-
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- $for C in range(0, CHANNEL_TILE, 16):
- vout${ABC[C:C+4]}${ABC[C+8:C+12]}${ABC[C+4:C+8]}${ABC[C+12:C+16]} = _mm256_min_epi16(_mm256_max_epi16(vout${ABC[C:C+4]}${ABC[C+8:C+12]}${ABC[C+4:C+8]}${ABC[C+12:C+16]}, voutput_min), voutput_max);
+ const __m256i vout${ABC[C:C+4]}${ABC[C+8:C+12]}${ABC[C+4:C+8]}${ABC[C+12:C+16]} = _mm256_adds_epi16(_mm256_packs_epi32(vacc${ABC[C:C+8]}, vacc${ABC[C+8:C+16]}), voutput_zero_point);
$for C in range(0, CHANNEL_TILE, 16):
__m128i vout${ABC[C:C+16]} = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout${ABC[C:C+4]}${ABC[C+8:C+12]}${ABC[C+4:C+8]}${ABC[C+12:C+16]}), _mm256_extracti128_si256(vout${ABC[C:C+4]}${ABC[C+8:C+12]}${ABC[C+4:C+8]}${ABC[C+12:C+16]}, 1)), _MM_SHUFFLE(3, 1, 2, 0));
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ $for C in range(0, CHANNEL_TILE, 16):
+ vout${ABC[C:C+16]} = _mm_max_epi8(vout${ABC[C:C+16]}, voutput_min);
+ vout${ABC[C:C+16]} = _mm_min_epi8(vout${ABC[C:C+16]}, voutput_max);
+
_mm_storeu_si128((__m128i*) output, vout${ABC[0:16]});
$for C in range(16, CHANNEL_TILE, 16):
_mm_storeu_si128((__m128i*) (output + ${C}), vout${ABC[C:C+16]});
@@ -137,8 +141,8 @@
w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t));
k += 16;
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc${ABC[1:8:2]} = _mm256_shuffle_epi32(vacc${ABC[0:8]}, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc${ABC[9:16:2]} = _mm256_shuffle_epi32(vacc${ABC[8:16]}, _MM_SHUFFLE(3, 3, 1, 1));
@@ -156,29 +160,28 @@
const __m256i vq31prod${ABC[0:8]} = _mm256_blend_epi16(vq31prod${ABC[0:8:2]}, vq31prod${ABC[1:8:2]}, 0xCC);
const __m256i vq31prod${ABC[8:16]} = _mm256_blend_epi16(vq31prod${ABC[8:16:2]}, vq31prod${ABC[9:16:2]}, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem${ABC[0:8]} =
_mm256_add_epi32(_mm256_and_si256(vq31prod${ABC[0:8]}, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${ABC[0:8]}));
const __m256i vrem${ABC[8:16]} =
_mm256_add_epi32(_mm256_and_si256(vq31prod${ABC[8:16]}, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${ABC[8:16]}));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
vacc${ABC[0:8]} =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod${ABC[0:8]}, vshift), _mm256_cmpgt_epi32(vrem${ABC[0:8]}, vremainder_threshold));
vacc${ABC[8:16]} =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod${ABC[8:16]}, vshift), _mm256_cmpgt_epi32(vrem${ABC[8:16]}, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout${ABC[0:8]} = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc${ABC[0:8]}), _mm256_extracti128_si256(vacc${ABC[0:8]}, 1)), voutput_zero_point);
__m128i vout${ABC[8:16]} = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc${ABC[8:16]}), _mm256_extracti128_si256(vacc${ABC[8:16]}, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout${ABC[0:8]} = _mm_min_epi16(_mm_max_epi16(vout${ABC[0:8]}, voutput_min), voutput_max);
- vout${ABC[8:16]} = _mm_min_epi16(_mm_max_epi16(vout${ABC[8:16]}, voutput_min), voutput_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
__m128i vout${ABC[0:16]} = _mm_packs_epi16(vout${ABC[0:8]}, vout${ABC[8:16]});
+ vout${ABC[0:16]} = _mm_min_epi8(_mm_max_epi8(vout${ABC[0:16]}, voutput_min), voutput_max);
$if CHANNEL_TILE > 16:
if XNN_LIKELY(c >= 16) {
diff --git a/src/qs8-dwconv/unipass-avx2-mul32.c.in b/src/qs8-dwconv/unipass-avx2-mul32.c.in
index 41086fd..8d808fc 100644
--- a/src/qs8-dwconv/unipass-avx2-mul32.c.in
+++ b/src/qs8-dwconv/unipass-avx2-mul32.c.in
@@ -60,8 +60,8 @@
w = (const void*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(int32_t) + ${KERNEL_TILE * CHANNEL_TILE} * sizeof(int8_t));
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
$for C in range(0, CHANNEL_TILE, 8):
const __m256i vacc${ABC[C+1:C+8:2]} = _mm256_shuffle_epi32(vacc${ABC[C:C+8]}, _MM_SHUFFLE(3, 3, 1, 1));
@@ -77,21 +77,24 @@
$for C in range(0, CHANNEL_TILE, 8):
const __m256i vq31prod${ABC[C:C+8]} = _mm256_blend_epi16(vq31prod${ABC[C:C+8:2]}, vq31prod${ABC[C+1:C+8:2]}, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
$for C in range(0, CHANNEL_TILE, 8):
const __m256i vrem${ABC[C:C+8]} =
_mm256_add_epi32(_mm256_and_si256(vq31prod${ABC[C:C+8]}, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${ABC[C:C+8]}));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ $if CHANNEL_TILE > 8:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
$for C in range(0, CHANNEL_TILE, 8):
vacc${ABC[C:C+8]} =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod${ABC[C:C+8]}, vshift), _mm256_cmpgt_epi32(vrem${ABC[C:C+8]}, vremainder_threshold));
$if CHANNEL_TILE > 8:
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
$else:
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
$for C in range(0, CHANNEL_TILE, 16):
$if C + 8 < CHANNEL_TILE:
__m256i vout${ABC[C:C+4]}${ABC[C+8:C+12]}${ABC[C+4:C+8]}${ABC[C+12:C+16]} = _mm256_adds_epi16(_mm256_packs_epi32(vacc${ABC[C:C+8]}, vacc${ABC[C+8:C+16]}), voutput_zero_point);
@@ -100,26 +103,22 @@
$else:
__m128i vout${ABC[C:C+8]} = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc${ABC[C:C+8]}), _mm256_extracti128_si256(vacc${ABC[C:C+8]}, 1)), voutput_zero_point);
- $if CHANNEL_TILE > 8:
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- $else:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- $for C in range(0, CHANNEL_TILE, 16):
- $if C + 8 < CHANNEL_TILE:
- vout${ABC[C:C+4]}${ABC[C+8:C+12]}${ABC[C+4:C+8]}${ABC[C+12:C+16]} = _mm256_min_epi16(_mm256_max_epi16(vout${ABC[C:C+4]}${ABC[C+8:C+12]}${ABC[C+4:C+8]}${ABC[C+12:C+16]}, voutput_min), voutput_max);
- $elif CHANNEL_TILE > 8:
- vout${ABC[C:C+8]} = _mm_min_epi16(_mm_max_epi16(vout${ABC[C:C+8]}, _mm256_castsi256_si128(voutput_min)), _mm256_castsi256_si128(voutput_max));
- $else:
- vout${ABC[C:C+8]} = _mm_min_epi16(_mm_max_epi16(vout${ABC[C:C+8]}, voutput_min), voutput_max);
-
$for C in range(0, CHANNEL_TILE, 16):
$if C + 8 < CHANNEL_TILE:
__m128i vout${ABC[C:C+16]} = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout${ABC[C:C+4]}${ABC[C+8:C+12]}${ABC[C+4:C+8]}${ABC[C+12:C+16]}), _mm256_extracti128_si256(vout${ABC[C:C+4]}${ABC[C+8:C+12]}${ABC[C+4:C+8]}${ABC[C+12:C+16]}, 1)), _MM_SHUFFLE(3, 1, 2, 0));
$else:
__m128i vout${ABC[C:C+8]}${ABC[C:C+8]} = _mm_packs_epi16(vout${ABC[C:C+8]}, vout${ABC[C:C+8]});
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ $for C in range(0, CHANNEL_TILE, 16):
+ $if C + 8 < CHANNEL_TILE:
+ vout${ABC[C:C+16]} = _mm_max_epi8(vout${ABC[C:C+16]}, voutput_min);
+ vout${ABC[C:C+16]} = _mm_min_epi8(vout${ABC[C:C+16]}, voutput_max);
+ $else:
+ vout${ABC[C:C+8]}${ABC[C:C+8]} = _mm_max_epi8(vout${ABC[C:C+8]}${ABC[C:C+8]}, voutput_min);
+ vout${ABC[C:C+8]}${ABC[C:C+8]} = _mm_min_epi8(vout${ABC[C:C+8]}${ABC[C:C+8]}, voutput_max);
+
$if CHANNEL_TILE > 8:
_mm_storeu_si128((__m128i*) output, vout${ABC[0:16]});
$else:
@@ -156,8 +155,8 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc${ABC[1:8:2]} = _mm256_shuffle_epi32(vacc${ABC[0:8]}, _MM_SHUFFLE(3, 3, 1, 1));
@@ -169,23 +168,23 @@
const __m256i vq31prod${ABC[0:8]} = _mm256_blend_epi16(vq31prod${ABC[0:8:2]}, vq31prod${ABC[1:8:2]}, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem${ABC[0:8]} =
_mm256_add_epi32(_mm256_and_si256(vq31prod${ABC[0:8]}, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${ABC[0:8]}));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc${ABC[0:8]} =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod${ABC[0:8]}, vshift), _mm256_cmpgt_epi32(vrem${ABC[0:8]}, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->avx2.output_zero_point);
__m128i vout${ABC[0:8]} = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc${ABC[0:8]}), _mm256_extracti128_si256(vacc${ABC[0:8]}, 1)), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout${ABC[0:8]} = _mm_min_epi16(_mm_max_epi16(vout${ABC[0:8]}, voutput_min), voutput_max);
-
__m128i vout${ABC[0:8]}${ABC[0:8]} = _mm_packs_epi16(vout${ABC[0:8]}, vout${ABC[0:8]});
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx2.output_min);
+ vout${ABC[0:8]}${ABC[0:8]} = _mm_min_epi8(vout${ABC[0:8]}${ABC[0:8]}, voutput_max);
+ vout${ABC[0:8]}${ABC[0:8]} = _mm_max_epi8(vout${ABC[0:8]}${ABC[0:8]}, voutput_min);
$if CHANNEL_TILE > 8:
if XNN_LIKELY(c >= 8) {
diff --git a/src/qs8-dwconv/unipass-sse-mul16.c.in b/src/qs8-dwconv/unipass-sse-mul16.c.in
index d53528b..5036dd7 100644
--- a/src/qs8-dwconv/unipass-sse-mul16.c.in
+++ b/src/qs8-dwconv/unipass-sse-mul16.c.in
@@ -15,6 +15,7 @@
#include <xnnpack/dwconv.h>
+$PARAMS_STRUCT = "sse4" if SSE >= 4 else "sse2"
$ISA = "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
void xnn_qs8_dwconv_minmax_ukernel_up${CHANNEL_TILE}x${KERNEL_TILE}__${ISA}_mul16(
size_t channels,
@@ -76,8 +77,8 @@
w = (const void*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(int32_t) + ${KERNEL_TILE * CHANNEL_TILE} * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
$if SSE >= 4:
$for C in range(0, CHANNEL_TILE, 4):
@@ -125,25 +126,29 @@
$for C in range(0, CHANNEL_TILE, 4):
const __m128i vq31prod${ABC[C:C+4]} = _mm_shuffle_epi32(vq31prod${ABC[C:C+4:2]}${ABC[C+1:C+4:2]}, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
$for C in range(0, CHANNEL_TILE, 4):
const __m128i vrem${ABC[C:C+4]} =
_mm_add_epi32(_mm_and_si128(vq31prod${ABC[C:C+4]}, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${ABC[C:C+4]}));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
$for C in range(0, CHANNEL_TILE, 4):
vacc${ABC[C:C+4]} =
_mm_sub_epi32(_mm_sra_epi32(vq31prod${ABC[C:C+4]}, vshift), _mm_cmpgt_epi32(vrem${ABC[C:C+4]}, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
$for C in range(0, CHANNEL_TILE, 8):
__m128i vout${ABC[C:C+8]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[C:C+4]}, vacc${ABC[C+4:C+8]}), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- $for C in range(0, CHANNEL_TILE, 8):
- vout${ABC[C:C+8]} = _mm_min_epi16(_mm_max_epi16(vout${ABC[C:C+8]}, voutput_min), voutput_max);
+ $if SSE < 4:
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ $for C in range(0, CHANNEL_TILE, 8):
+ vout${ABC[C:C+8]} = _mm_max_epi16(vout${ABC[C:C+8]}, voutput_min);
+
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ $for C in range(0, CHANNEL_TILE, 8):
+ vout${ABC[C:C+8]} = _mm_min_epi16(vout${ABC[C:C+8]}, voutput_max);
$for C in range(0, CHANNEL_TILE, 16):
$if C + 8 < CHANNEL_TILE:
@@ -151,6 +156,21 @@
$else:
__m128i vout${ABC[C:C+8]}${ABC[C:C+8]} = _mm_packs_epi16(vout${ABC[C:C+8]}, vout${ABC[C:C+8]});
+ $if SSE == 4:
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ $for C in range(0, CHANNEL_TILE, 16):
+ $if C + 8 < CHANNEL_TILE:
+ vout${ABC[C:C+16]} = _mm_max_epi8(vout${ABC[C:C+16]}, voutput_min);
+ $else:
+ vout${ABC[C:C+8]}${ABC[C:C+8]} = _mm_max_epi8(vout${ABC[C:C+8]}${ABC[C:C+8]}, voutput_min);
+
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
+ $for C in range(0, CHANNEL_TILE, 16):
+ $if C + 8 < CHANNEL_TILE:
+ vout${ABC[C:C+16]} = _mm_min_epi8(vout${ABC[C:C+16]}, voutput_max);
+ $else:
+ vout${ABC[C:C+8]}${ABC[C:C+8]} = _mm_min_epi8(vout${ABC[C:C+8]}${ABC[C:C+8]}, voutput_max);
+
$if CHANNEL_TILE > 8:
_mm_storeu_si128((__m128i*) output, vout${ABC[0:16]});
$else:
@@ -200,10 +220,10 @@
w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
k += 8;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
- $if SSE >= 4:
+ $if SSE == 4:
const __m128i vacc${ABC[1:4:2]} = _mm_shuffle_epi32(vacc${ABC[0:4]}, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc${ABC[5:8:2]} = _mm_shuffle_epi32(vacc${ABC[4:8]}, _MM_SHUFFLE(3, 3, 1, 1));
@@ -262,28 +282,32 @@
const __m128i vq31prod${ABC[0:4]} = _mm_shuffle_epi32(vq31prod${ABC[0:4:2]}${ABC[1:4:2]}, _MM_SHUFFLE(3, 1, 2, 0));
const __m128i vq31prod${ABC[4:8]} = _mm_shuffle_epi32(vq31prod${ABC[4:8:2]}${ABC[5:8:2]}, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
const __m128i vrem${ABC[0:4]} =
_mm_add_epi32(_mm_and_si128(vq31prod${ABC[0:4]}, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${ABC[0:4]}));
const __m128i vrem${ABC[4:8]} =
_mm_add_epi32(_mm_and_si128(vq31prod${ABC[4:8]}, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${ABC[4:8]}));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
vacc${ABC[0:4]} =
_mm_sub_epi32(_mm_sra_epi32(vq31prod${ABC[0:4]}, vshift), _mm_cmpgt_epi32(vrem${ABC[0:4]}, vremainder_threshold));
vacc${ABC[4:8]} =
_mm_sub_epi32(_mm_sra_epi32(vq31prod${ABC[4:8]}, vshift), _mm_cmpgt_epi32(vrem${ABC[4:8]}, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
__m128i vout${ABC[0:8]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[0:4]}, vacc${ABC[4:8]}), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout${ABC[0:8]} = _mm_min_epi16(_mm_max_epi16(vout${ABC[0:8]}, voutput_min), voutput_max);
+ $if SSE < 4:
+ vout${ABC[0:8]} = _mm_max_epi16(vout${ABC[0:8]}, _mm_load_si128((const __m128i*) params->sse2.output_min));
+ vout${ABC[0:8]} = _mm_min_epi16(vout${ABC[0:8]}, _mm_load_si128((const __m128i*) params->sse2.output_max));
__m128i vout${ABC[0:8]}${ABC[0:8]} = _mm_packs_epi16(vout${ABC[0:8]}, vout${ABC[0:8]});
+ $if SSE == 4:
+ vout${ABC[0:8]}${ABC[0:8]} = _mm_max_epi8(vout${ABC[0:8]}${ABC[0:8]}, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout${ABC[0:8]}${ABC[0:8]} = _mm_min_epi8(vout${ABC[0:8]}${ABC[0:8]}, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
$if CHANNEL_TILE > 8:
if XNN_LIKELY(c >= 8) {
_mm_storel_epi64((__m128i*) output, vout${ABC[0:8]}${ABC[0:8]});
diff --git a/src/qs8-dwconv/unipass-sse-mul32.c.in b/src/qs8-dwconv/unipass-sse-mul32.c.in
index 73e1332..739501c 100644
--- a/src/qs8-dwconv/unipass-sse-mul32.c.in
+++ b/src/qs8-dwconv/unipass-sse-mul32.c.in
@@ -75,8 +75,8 @@
w = (const void*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(int32_t) + ${KERNEL_TILE * CHANNEL_TILE} * sizeof(int8_t));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
$for C in range(0, CHANNEL_TILE, 4):
const __m128i vacc${ABC[C+1:C+4:2]} = _mm_shuffle_epi32(vacc${ABC[C:C+4]}, _MM_SHUFFLE(3, 3, 1, 1));
@@ -90,31 +90,32 @@
$for C in range(0, CHANNEL_TILE, 4):
const __m128i vq31prod${ABC[C:C+4]} = _mm_blend_epi16(vq31prod${ABC[C:C+4:2]}, vq31prod${ABC[C+1:C+4:2]}, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
$for C in range(0, CHANNEL_TILE, 4):
const __m128i vrem${ABC[C:C+4]} =
_mm_add_epi32(_mm_and_si128(vq31prod${ABC[C:C+4]}, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${ABC[C:C+4]}));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
$for C in range(0, CHANNEL_TILE, 4):
vacc${ABC[C:C+4]} =
_mm_sub_epi32(_mm_sra_epi32(vq31prod${ABC[C:C+4]}, vshift), _mm_cmpgt_epi32(vrem${ABC[C:C+4]}, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
$for C in range(0, CHANNEL_TILE, 8):
__m128i vout${ABC[C:C+8]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[C:C+4]}, vacc${ABC[C+4:C+8]}), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- $for C in range(0, CHANNEL_TILE, 8):
- vout${ABC[C:C+8]} = _mm_min_epi16(_mm_max_epi16(vout${ABC[C:C+8]}, voutput_min), voutput_max);
-
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
$for C in range(0, CHANNEL_TILE, 16):
$if C + 8 < CHANNEL_TILE:
__m128i vout${ABC[C:C+16]} = _mm_packs_epi16(vout${ABC[C:C+8]}, vout${ABC[C+8:C+16]});
+ vout${ABC[C:C+16]} = _mm_max_epi8(vout${ABC[C:C+16]}, voutput_min);
+ vout${ABC[C:C+16]} = _mm_min_epi8(vout${ABC[C:C+16]}, voutput_max);
$else:
__m128i vout${ABC[C:C+8]}${ABC[C:C+8]} = _mm_packs_epi16(vout${ABC[C:C+8]}, vout${ABC[C:C+8]});
+ vout${ABC[C:C+8]}${ABC[C:C+8]} = _mm_max_epi8(vout${ABC[C:C+8]}${ABC[C:C+8]}, voutput_min);
+ vout${ABC[C:C+8]}${ABC[C:C+8]} = _mm_min_epi8(vout${ABC[C:C+8]}${ABC[C:C+8]}, voutput_max);
$if CHANNEL_TILE > 8:
_mm_storeu_si128((__m128i*) output, vout${ABC[0:16]});
@@ -155,8 +156,8 @@
w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
k += 4;
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc${ABC[1:4:2]} = _mm_shuffle_epi32(vacc${ABC[0:4]}, _MM_SHUFFLE(3, 3, 1, 1));
@@ -168,23 +169,21 @@
const __m128i vq31prod${ABC[0:4]} = _mm_blend_epi16(vq31prod${ABC[0:4:2]}, vq31prod${ABC[1:4:2]}, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem${ABC[0:4]} =
_mm_add_epi32(_mm_and_si128(vq31prod${ABC[0:4]}, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${ABC[0:4]}));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc${ABC[0:4]} =
_mm_sub_epi32(_mm_sra_epi32(vq31prod${ABC[0:4]}, vshift), _mm_cmpgt_epi32(vrem${ABC[0:4]}, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout${ABC[0:4]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[0:4]}, vacc${ABC[0:4]}), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vout${ABC[0:4]} = _mm_min_epi16(_mm_max_epi16(vout${ABC[0:4]}, voutput_min), voutput_max);
-
vout${ABC[0:4]} = _mm_packs_epi16(vout${ABC[0:4]}, vout${ABC[0:4]});
+ vout${ABC[0:4]} = _mm_max_epi8(vout${ABC[0:4]}, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout${ABC[0:4]} = _mm_min_epi8(vout${ABC[0:4]}, _mm_load_si128((const __m128i*) params->sse4.output_max));
$if CHANNEL_TILE > 4:
if XNN_LIKELY(c >= 4) {
diff --git a/src/qs8-gemm/MRx16c8-avx512skx.c.in b/src/qs8-gemm/MRx16c8-avx512skx.c.in
index adcbe21..7870b6c 100644
--- a/src/qs8-gemm/MRx16c8-avx512skx.c.in
+++ b/src/qs8-gemm/MRx16c8-avx512skx.c.in
@@ -61,19 +61,24 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
$if MR > 1:
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
$else:
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ $if MR > 2:
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ $elif MR == 2:
+ const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_min));
+ const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ $else:
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
$for N in range(4, 16, 4):
@@ -147,26 +152,28 @@
_mm512_mask_sub_epi32(vacc${M}x084C195D2A6E3B7F, _mm512_cmpgt_epi32_mask(vrem${M}x084C195D2A6E3B7F, vremainder_threshold), vacc${M}x084C195D2A6E3B7F, vminus_one);
$if MR == 1:
- __m256i vacc0x084C2A6E195D3B7F = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc0x084C195D2A6E3B7F), _mm512_extracti32x8_epi32(vacc0x084C195D2A6E3B7F, 1)), voutput_zero_point);
- vacc0x084C2A6E195D3B7F = _mm256_min_epi16(_mm256_max_epi16(vacc0x084C2A6E195D3B7F, voutput_min), voutput_max);
+ const __m256i vacc0x084C2A6E195D3B7F = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc0x084C195D2A6E3B7F), _mm512_extracti32x8_epi32(vacc0x084C195D2A6E3B7F, 1)), voutput_zero_point);
$else:
$for M in range(0, MR, 2):
- __m512i vacc${M}${min(M+1, MR-1)}x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc${M}x084C195D2A6E3B7F, vacc${min(M+1, MR-1)}x084C195D2A6E3B7F), voutput_zero_point);
-
- $for M in range(0, MR, 2):
- vacc${M}${min(M+1, MR-1)}x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc${M}${min(M+1, MR-1)}x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
+ const __m512i vacc${M}${min(M+1, MR-1)}x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc${M}x084C195D2A6E3B7F, vacc${min(M+1, MR-1)}x084C195D2A6E3B7F), voutput_zero_point);
$if MR > 2:
__m512i vout012${min(3, MR-1)}x084Cx195Dx2A6Ex3B7F = _mm512_packs_epi16(vacc01x084Cx195Dx2A6Ex3B7F, vacc2${min(3, MR-1)}x084Cx195Dx2A6Ex3B7F);
vout012${min(M+3, MR-1)}x084Cx195Dx2A6Ex3B7F = _mm512_permutexvar_epi32(_mm512_set_epi32(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0), vout012${min(3, MR-1)}x084Cx195Dx2A6Ex3B7F);
- const __m512i vout012${min(3, MR-1)}x0123456789ABCDEF = _mm512_shuffle_epi8(vout012${min(3, MR-1)}x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ __m512i vout012${min(3, MR-1)}x0123456789ABCDEF = _mm512_shuffle_epi8(vout012${min(3, MR-1)}x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ vout012${min(3, MR-1)}x0123456789ABCDEF = _mm512_max_epi8(vout012${min(3, MR-1)}x0123456789ABCDEF, voutput_min);
+ vout012${min(3, MR-1)}x0123456789ABCDEF = _mm512_min_epi8(vout012${min(3, MR-1)}x0123456789ABCDEF, voutput_max);
$elif MR == 2:
const __m256i vout01x084Cx2A6Ex195Dx3B7F = _mm256_packs_epi16(_mm512_castsi512_si256(vacc01x084Cx195Dx2A6Ex3B7F), _mm512_extracti32x8_epi32(vacc01x084Cx195Dx2A6Ex3B7F, 1));
const __m256i vout01x084C2A6E195D3B7F = _mm256_permutevar8x32_epi32(vout01x084Cx2A6Ex195Dx3B7F, _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0));
- const __m256i vout01x0123456789ABCDEF = _mm256_shuffle_epi8(vout01x084C2A6E195D3B7F, _mm256_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0, 15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ __m256i vout01x0123456789ABCDEF = _mm256_shuffle_epi8(vout01x084C2A6E195D3B7F, _mm256_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0, 15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ vout01x0123456789ABCDEF = _mm256_max_epi8(vout01x0123456789ABCDEF, voutput_min);
+ vout01x0123456789ABCDEF = _mm256_min_epi8(vout01x0123456789ABCDEF, voutput_max);
$elif MR == 1:
const __m128i vout0x084C2A6E195D3B7F = _mm_packs_epi16(_mm256_castsi256_si128(vacc0x084C2A6E195D3B7F), _mm256_extracti128_si256(vacc0x084C2A6E195D3B7F, 1));
- const __m128i vout0x0123456789ABCDEF = _mm_shuffle_epi8(vout0x084C2A6E195D3B7F, _mm_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ __m128i vout0x0123456789ABCDEF = _mm_shuffle_epi8(vout0x084C2A6E195D3B7F, _mm_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ vout0x0123456789ABCDEF = _mm_max_epi8(vout0x0123456789ABCDEF, voutput_min);
+ vout0x0123456789ABCDEF = _mm_min_epi8(vout0x0123456789ABCDEF, voutput_max);
$if MR > 2:
if (nc >= 16) {
diff --git a/src/qs8-gemm/MRx4c2-sse.c.in b/src/qs8-gemm/MRx4c2-sse.c.in
index 9e25af1..7669bfa 100644
--- a/src/qs8-gemm/MRx4c2-sse.c.in
+++ b/src/qs8-gemm/MRx4c2-sse.c.in
@@ -27,6 +27,7 @@
$LOAD_SUFFIX = {"LD128": "_ld128", "LD64": "_ld64", "EXTENDED": ""}[VARIANT]
$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else ""
+$PARAMS_STRUCT = "sse4" if SSE >= 4 else "sse2"
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_ukernel_${MR}x4c2__${ISA}${LOAD_SUFFIX}(
size_t mr,
@@ -214,8 +215,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
$if SSE == 4:
$for M in range(MR):
@@ -277,31 +278,39 @@
$for M in range(MR):
const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
$for M in range(MR):
const __m128i vrem${M}x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
$for M in range(MR):
vacc${M}x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
$for M in range(0, MR, 2):
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- $for M in range(0, MR, 2):
- vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
+ $if SSE < 4:
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ $for M in range(0, MR, 2):
+ vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
$if MR > 2:
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc${min(2, MR-1)}${min(3, MR-1)}x0123);
$else:
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
+ $if SSE == 4:
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
$for M in range(1, MR):
diff --git a/src/qs8-gemm/MRx4c8-sse.c.in b/src/qs8-gemm/MRx4c8-sse.c.in
index 46a0fff..0f2ed51 100644
--- a/src/qs8-gemm/MRx4c8-sse.c.in
+++ b/src/qs8-gemm/MRx4c8-sse.c.in
@@ -27,6 +27,7 @@
$LOAD_SUFFIX = {"LD128": "_ld128", "LD64": "_ld64", "EXTENDED": ""}[VARIANT]
$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else ""
+$PARAMS_STRUCT = "sse4" if SSE >= 4 else "sse2"
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_ukernel_${MR}x4c8__${ISA}${LOAD_SUFFIX}(
size_t mr,
@@ -151,8 +152,8 @@
$for M in range(MR):
__m128i vacc${M}x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc${M}x02, vacc${M}x13), _mm_unpackhi_epi32(vacc${M}x02, vacc${M}x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
$if SSE == 4:
$for M in range(MR):
@@ -214,31 +215,39 @@
$for M in range(MR):
const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
$for M in range(MR):
const __m128i vrem${M}x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
$for M in range(MR):
vacc${M}x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
$for M in range(0, MR, 2):
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- $for M in range(0, MR, 2):
- vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
+ $if SSE < 4:
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_max);
+ $for M in range(0, MR, 2):
+ vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
$if MR > 2:
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc${min(2, MR-1)}${min(3, MR-1)}x0123);
$else:
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
+ $if SSE == 4:
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
$for M in range(1, MR):
diff --git a/src/qs8-gemm/MRx8c8-avx2.c.in b/src/qs8-gemm/MRx8c8-avx2.c.in
index 70e4d32..bac1ce2 100644
--- a/src/qs8-gemm/MRx8c8-avx2.c.in
+++ b/src/qs8-gemm/MRx8c8-avx2.c.in
@@ -112,11 +112,11 @@
$for M in range(MR):
__m256i vacc${M}x01234567 = _mm256_permutevar8x32_epi32(vacc${M}x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
$for M in range(MR):
- const __m256i vacc${M}x11335577 = _mm256_shuffle_epi32(vacc${M}x01234567, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m256i vacc${M}x11335577 = _mm256_srli_epi64(vacc${M}x01234567, 32);
$for M in range(MR):
const __m256i vprod${M}x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc${M}x01234567, vmultiplier), vrounding);
@@ -131,33 +131,35 @@
$for M in range(MR):
const __m256i vq31prod${M}x01234567 = _mm256_blend_epi16(vq31prod${M}x0246, vq31prod${M}x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
$for M in range(MR):
const __m256i vrem${M}x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod${M}x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${M}x01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
$for M in range(MR):
vacc${M}x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod${M}x01234567, vshift), _mm256_cmpgt_epi32(vrem${M}x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
$for M in range(0, MR, 2):
__m256i vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc${M}x01234567, vacc${min(M+1, MR-1)}x01234567), voutput_zero_point);
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_permute4x64_epi64(vacc${M}${min(M+1, MR-1)}x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- $for M in range(0, MR, 2):
- vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc${M}${min(M+1, MR-1)}x01234567, voutput_min), voutput_max);
-
$if MR > 2:
__m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc${min(2, MR-1)}${min(3, MR-1)}x01234567);
$else:
__m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/1x16c8-minmax-avx512skx.c b/src/qs8-gemm/gen/1x16c8-minmax-avx512skx.c
index 412da85..7729f91 100644
--- a/src/qs8-gemm/gen/1x16c8-minmax-avx512skx.c
+++ b/src/qs8-gemm/gen/1x16c8-minmax-avx512skx.c
@@ -43,14 +43,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
@@ -105,11 +105,12 @@
vacc0x084C195D2A6E3B7F =
_mm512_mask_sub_epi32(vacc0x084C195D2A6E3B7F, _mm512_cmpgt_epi32_mask(vrem0x084C195D2A6E3B7F, vremainder_threshold), vacc0x084C195D2A6E3B7F, vminus_one);
- __m256i vacc0x084C2A6E195D3B7F = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc0x084C195D2A6E3B7F), _mm512_extracti32x8_epi32(vacc0x084C195D2A6E3B7F, 1)), voutput_zero_point);
- vacc0x084C2A6E195D3B7F = _mm256_min_epi16(_mm256_max_epi16(vacc0x084C2A6E195D3B7F, voutput_min), voutput_max);
+ const __m256i vacc0x084C2A6E195D3B7F = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc0x084C195D2A6E3B7F), _mm512_extracti32x8_epi32(vacc0x084C195D2A6E3B7F, 1)), voutput_zero_point);
const __m128i vout0x084C2A6E195D3B7F = _mm_packs_epi16(_mm256_castsi256_si128(vacc0x084C2A6E195D3B7F), _mm256_extracti128_si256(vacc0x084C2A6E195D3B7F, 1));
- const __m128i vout0x0123456789ABCDEF = _mm_shuffle_epi8(vout0x084C2A6E195D3B7F, _mm_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ __m128i vout0x0123456789ABCDEF = _mm_shuffle_epi8(vout0x084C2A6E195D3B7F, _mm_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ vout0x0123456789ABCDEF = _mm_max_epi8(vout0x0123456789ABCDEF, voutput_min);
+ vout0x0123456789ABCDEF = _mm_min_epi8(vout0x0123456789ABCDEF, voutput_max);
if (nc >= 16) {
_mm_storeu_si128((__m128i*) c0, vout0x0123456789ABCDEF);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-avx-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-avx-ld128.c
index b02cf51..8350dc1 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-avx-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-avx-ld128.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,24 +119,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-avx-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-avx-ld64.c
index 29a3338..63362ee 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-avx-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-avx-ld64.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,24 +119,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-sse2-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-sse2-ld128.c
index f641df6..7df288a 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-sse2-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-sse2-ld128.c
@@ -153,6 +153,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-sse2-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-sse2-ld64.c
index b8af56f..d809fb1 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-sse2-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-sse2-ld64.c
@@ -153,6 +153,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-sse41-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-sse41-ld128.c
index 67226d8..6b57e90 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-sse41-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-sse41-ld128.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,24 +119,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-sse41-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-sse41-ld64.c
index 31bd480..5a75904 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-sse41-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-sse41-ld64.c
@@ -105,8 +105,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,24 +119,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-ssse3-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-ssse3-ld128.c
index 064aab2..b3ea64b 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-ssse3-ld128.c
@@ -153,6 +153,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-ssse3-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-ssse3-ld64.c
index 5065d6f..1401dc4 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-ssse3-ld64.c
@@ -153,6 +153,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-xop-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-xop-ld128.c
index a208aeb..227d0ab 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-xop-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-xop-ld128.c
@@ -110,8 +110,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -124,24 +124,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-xop-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-xop-ld64.c
index eba8477..1a3f597 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-xop-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-xop-ld64.c
@@ -110,8 +110,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -124,24 +124,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-avx.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-avx.c
index 48b7e65..4c04278 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-avx.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-avx.c
@@ -98,8 +98,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -112,24 +112,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-sse2.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-sse2.c
index 23fd99e..299d609 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-sse2.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-sse2.c
@@ -146,6 +146,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-sse41.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-sse41.c
index 9ea9ad4..975d67c 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-sse41.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-sse41.c
@@ -98,8 +98,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -112,24 +112,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-ssse3.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-ssse3.c
index 6a9b744..1b56035 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-ssse3.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-ssse3.c
@@ -146,6 +146,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-xop.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-xop.c
index b4d11ab..5565473 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-xop.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-xop.c
@@ -103,8 +103,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -117,24 +117,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-avx-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-avx-ld128.c
index 795a68a..9f3ee44 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-avx-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-avx-ld128.c
@@ -77,8 +77,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -91,24 +91,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-avx-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-avx-ld64.c
index 77d1188..d537eeb 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-avx-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-avx-ld64.c
@@ -79,8 +79,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -93,24 +93,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-sse2-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-sse2-ld128.c
index b8afcda..6370331 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-sse2-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-sse2-ld128.c
@@ -125,6 +125,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-sse2-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-sse2-ld64.c
index 862a357..89adced 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-sse2-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-sse2-ld64.c
@@ -127,6 +127,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-sse41-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-sse41-ld128.c
index 49213c7..a68701d 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-sse41-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-sse41-ld128.c
@@ -77,8 +77,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -91,24 +91,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-sse41-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-sse41-ld64.c
index b137d9d..17a890e 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-sse41-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-sse41-ld64.c
@@ -79,8 +79,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -93,24 +93,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-ssse3-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-ssse3-ld128.c
index a429fef..33104b1 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-ssse3-ld128.c
@@ -125,6 +125,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-ssse3-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-ssse3-ld64.c
index f3f96c5..535b406 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-ssse3-ld64.c
@@ -127,6 +127,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-xop-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-xop-ld128.c
index 169e070..34e3cb0 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-xop-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-xop-ld128.c
@@ -82,8 +82,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -96,24 +96,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-xop-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-xop-ld64.c
index cfbd035..ebcc4aa 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-xop-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-xop-ld64.c
@@ -84,8 +84,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -98,24 +98,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-avx.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-avx.c
index 29b9b6b..6d4827c 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-avx.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-avx.c
@@ -75,8 +75,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -89,24 +89,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-sse2.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-sse2.c
index 177f057..c5da5d5 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-sse2.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-sse2.c
@@ -123,6 +123,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-sse41.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-sse41.c
index 2ce36cb..c0ee9fd 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-sse41.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-sse41.c
@@ -75,8 +75,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -89,24 +89,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-ssse3.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-ssse3.c
index 57abe7a..9967e0f 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-ssse3.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-ssse3.c
@@ -123,6 +123,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-xop.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-xop.c
index 8df261f..44bc8cc 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-xop.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-xop.c
@@ -80,8 +80,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -94,24 +94,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x8c8-minmax-avx2.c b/src/qs8-gemm/gen/1x8c8-minmax-avx2.c
index d41d196..5f0a77c 100644
--- a/src/qs8-gemm/gen/1x8c8-minmax-avx2.c
+++ b/src/qs8-gemm/gen/1x8c8-minmax-avx2.c
@@ -91,10 +91,10 @@
const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
- const __m256i vacc0x11335577 = _mm256_shuffle_epi32(vacc0x01234567, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
const __m256i vprod0x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc0x01234567, vmultiplier), vrounding);
@@ -105,25 +105,25 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vacc00x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc0x01234567), voutput_zero_point);
vacc00x01234567 = _mm256_permute4x64_epi64(vacc00x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vacc00x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc00x01234567, voutput_min), voutput_max);
-
__m256i vout = _mm256_packs_epi16(vacc00x01234567, vacc00x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/1x8c8-xw-minmax-avx2.c b/src/qs8-gemm/gen/1x8c8-xw-minmax-avx2.c
index 74a33fe..69b6b7c 100644
--- a/src/qs8-gemm/gen/1x8c8-xw-minmax-avx2.c
+++ b/src/qs8-gemm/gen/1x8c8-xw-minmax-avx2.c
@@ -87,10 +87,10 @@
const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
- const __m256i vacc0x11335577 = _mm256_shuffle_epi32(vacc0x01234567, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
const __m256i vprod0x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc0x01234567, vmultiplier), vrounding);
@@ -101,25 +101,25 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vacc00x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc0x01234567), voutput_zero_point);
vacc00x01234567 = _mm256_permute4x64_epi64(vacc00x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vacc00x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc00x01234567, voutput_min), voutput_max);
-
__m256i vout = _mm256_packs_epi16(vacc00x01234567, vacc00x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/2x16c8-minmax-avx512skx.c b/src/qs8-gemm/gen/2x16c8-minmax-avx512skx.c
index 8b23c20..cab5fd0 100644
--- a/src/qs8-gemm/gen/2x16c8-minmax-avx512skx.c
+++ b/src/qs8-gemm/gen/2x16c8-minmax-avx512skx.c
@@ -49,14 +49,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_min));
+ const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
@@ -135,13 +135,13 @@
vacc1x084C195D2A6E3B7F =
_mm512_mask_sub_epi32(vacc1x084C195D2A6E3B7F, _mm512_cmpgt_epi32_mask(vrem1x084C195D2A6E3B7F, vremainder_threshold), vacc1x084C195D2A6E3B7F, vminus_one);
- __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
-
- vacc01x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc01x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
+ const __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
const __m256i vout01x084Cx2A6Ex195Dx3B7F = _mm256_packs_epi16(_mm512_castsi512_si256(vacc01x084Cx195Dx2A6Ex3B7F), _mm512_extracti32x8_epi32(vacc01x084Cx195Dx2A6Ex3B7F, 1));
const __m256i vout01x084C2A6E195D3B7F = _mm256_permutevar8x32_epi32(vout01x084Cx2A6Ex195Dx3B7F, _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0));
- const __m256i vout01x0123456789ABCDEF = _mm256_shuffle_epi8(vout01x084C2A6E195D3B7F, _mm256_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0, 15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ __m256i vout01x0123456789ABCDEF = _mm256_shuffle_epi8(vout01x084C2A6E195D3B7F, _mm256_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0, 15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ vout01x0123456789ABCDEF = _mm256_max_epi8(vout01x0123456789ABCDEF, voutput_min);
+ vout01x0123456789ABCDEF = _mm256_min_epi8(vout01x0123456789ABCDEF, voutput_max);
if (nc >= 16) {
_mm_storeu_si128((__m128i*) c0, _mm256_castsi256_si128(vout01x0123456789ABCDEF));
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-avx-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-avx-ld128.c
index 681db21..a6bb8cf 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-avx-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-avx-ld128.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -152,28 +152,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-avx-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-avx-ld64.c
index 7c738c1..2b72822 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-avx-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-avx-ld64.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -152,28 +152,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-sse2-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-sse2-ld128.c
index c6b83cc..c4ade3b 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-sse2-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-sse2-ld128.c
@@ -198,6 +198,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-sse2-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-sse2-ld64.c
index 8abfa22..672ce4a 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-sse2-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-sse2-ld64.c
@@ -198,6 +198,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-sse41-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-sse41-ld128.c
index 3decb45..60a7aa1 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-sse41-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-sse41-ld128.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -152,28 +152,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-sse41-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-sse41-ld64.c
index dbadda0..7f5ba39 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-sse41-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-sse41-ld64.c
@@ -132,8 +132,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -152,28 +152,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-ssse3-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-ssse3-ld128.c
index 8250c5e..53bf77c 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-ssse3-ld128.c
@@ -198,6 +198,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-ssse3-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-ssse3-ld64.c
index e645259..c2aec7c 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-ssse3-ld64.c
@@ -198,6 +198,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-xop-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-xop-ld128.c
index c5e92c0..6266b33 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-xop-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-xop-ld128.c
@@ -137,8 +137,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -157,28 +157,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-xop-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-xop-ld64.c
index 37ebc2c..a4f12a4 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-xop-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-xop-ld64.c
@@ -137,8 +137,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -157,28 +157,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-avx.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-avx.c
index fff7970..dad6b55 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-avx.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-avx.c
@@ -125,8 +125,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -145,28 +145,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-sse2.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-sse2.c
index f4283b2..4beb018 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-sse2.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-sse2.c
@@ -191,6 +191,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-sse41.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-sse41.c
index 3d3015d..a1575e9 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-sse41.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-sse41.c
@@ -125,8 +125,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -145,28 +145,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-ssse3.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-ssse3.c
index cd0e181..d51e490 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-ssse3.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-ssse3.c
@@ -191,6 +191,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-xop.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-xop.c
index 5138b63..19dcda2 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-xop.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-xop.c
@@ -130,8 +130,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -150,28 +150,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-avx-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-avx-ld128.c
index 2fc78af..fc48e65 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-avx-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-avx-ld128.c
@@ -97,8 +97,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -117,28 +117,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-avx-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-avx-ld64.c
index 60896cc..713a705 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-avx-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-avx-ld64.c
@@ -99,8 +99,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,28 +119,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-sse2-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-sse2-ld128.c
index fdb93c7..978e912 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-sse2-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-sse2-ld128.c
@@ -163,6 +163,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-sse2-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-sse2-ld64.c
index d7678a3..8d7725e 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-sse2-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-sse2-ld64.c
@@ -165,6 +165,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-sse41-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-sse41-ld128.c
index dde7a53..6236921 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-sse41-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-sse41-ld128.c
@@ -97,8 +97,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -117,28 +117,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-sse41-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-sse41-ld64.c
index e5c3e40..d9db58d 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-sse41-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-sse41-ld64.c
@@ -99,8 +99,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -119,28 +119,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-ssse3-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-ssse3-ld128.c
index b319c41..26c49c9 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-ssse3-ld128.c
@@ -163,6 +163,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-ssse3-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-ssse3-ld64.c
index 4ba4dde..163a9f8 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-ssse3-ld64.c
@@ -165,6 +165,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-xop-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-xop-ld128.c
index 4e303c0..8f2ec5e 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-xop-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-xop-ld128.c
@@ -102,8 +102,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -122,28 +122,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-xop-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-xop-ld64.c
index c430eaf..16bbdcc 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-xop-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-xop-ld64.c
@@ -104,8 +104,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -124,28 +124,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-avx.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-avx.c
index 59961d9..3a1a964 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-avx.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-avx.c
@@ -95,8 +95,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -115,28 +115,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-sse2.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-sse2.c
index 2c1ada6..7764dd8 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-sse2.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-sse2.c
@@ -161,6 +161,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-sse41.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-sse41.c
index febf10b..1903f6b 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-sse41.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-sse41.c
@@ -95,8 +95,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -115,28 +115,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-ssse3.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-ssse3.c
index 762b37d..bde1b65 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-ssse3.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-ssse3.c
@@ -161,6 +161,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-xop.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-xop.c
index 390558f..b15a8f9 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-xop.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-xop.c
@@ -100,8 +100,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -120,28 +120,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/2x8c8-minmax-avx2.c b/src/qs8-gemm/gen/2x8c8-minmax-avx2.c
index 736f0fd..be11da6 100644
--- a/src/qs8-gemm/gen/2x8c8-minmax-avx2.c
+++ b/src/qs8-gemm/gen/2x8c8-minmax-avx2.c
@@ -112,11 +112,11 @@
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
- const __m256i vacc0x11335577 = _mm256_shuffle_epi32(vacc0x01234567, _MM_SHUFFLE(3, 3, 1, 1));
- const __m256i vacc1x11335577 = _mm256_shuffle_epi32(vacc1x01234567, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
+ const __m256i vacc1x11335577 = _mm256_srli_epi64(vacc1x01234567, 32);
const __m256i vprod0x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc0x01234567, vmultiplier), vrounding);
const __m256i vprod1x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc1x01234567, vmultiplier), vrounding);
@@ -132,29 +132,29 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod1x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod1x01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod1x01234567, vshift), _mm256_cmpgt_epi32(vrem1x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vacc01x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc01x01234567, voutput_min), voutput_max);
-
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc01x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/2x8c8-xw-minmax-avx2.c b/src/qs8-gemm/gen/2x8c8-xw-minmax-avx2.c
index 71f7c04..f4e707b 100644
--- a/src/qs8-gemm/gen/2x8c8-xw-minmax-avx2.c
+++ b/src/qs8-gemm/gen/2x8c8-xw-minmax-avx2.c
@@ -108,11 +108,11 @@
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
- const __m256i vacc0x11335577 = _mm256_shuffle_epi32(vacc0x01234567, _MM_SHUFFLE(3, 3, 1, 1));
- const __m256i vacc1x11335577 = _mm256_shuffle_epi32(vacc1x01234567, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
+ const __m256i vacc1x11335577 = _mm256_srli_epi64(vacc1x01234567, 32);
const __m256i vprod0x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc0x01234567, vmultiplier), vrounding);
const __m256i vprod1x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc1x01234567, vmultiplier), vrounding);
@@ -128,29 +128,29 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod1x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod1x01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod1x01234567, vshift), _mm256_cmpgt_epi32(vrem1x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vacc01x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc01x01234567, voutput_min), voutput_max);
-
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc01x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/3x16c8-minmax-avx512skx.c b/src/qs8-gemm/gen/3x16c8-minmax-avx512skx.c
index 03a173c..b6d6a6d 100644
--- a/src/qs8-gemm/gen/3x16c8-minmax-avx512skx.c
+++ b/src/qs8-gemm/gen/3x16c8-minmax-avx512skx.c
@@ -55,14 +55,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
@@ -165,15 +165,14 @@
vacc2x084C195D2A6E3B7F =
_mm512_mask_sub_epi32(vacc2x084C195D2A6E3B7F, _mm512_cmpgt_epi32_mask(vrem2x084C195D2A6E3B7F, vremainder_threshold), vacc2x084C195D2A6E3B7F, vminus_one);
- __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
- __m512i vacc22x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc2x084C195D2A6E3B7F, vacc2x084C195D2A6E3B7F), voutput_zero_point);
-
- vacc01x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc01x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
- vacc22x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc22x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
+ const __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
+ const __m512i vacc22x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc2x084C195D2A6E3B7F, vacc2x084C195D2A6E3B7F), voutput_zero_point);
__m512i vout0122x084Cx195Dx2A6Ex3B7F = _mm512_packs_epi16(vacc01x084Cx195Dx2A6Ex3B7F, vacc22x084Cx195Dx2A6Ex3B7F);
vout0122x084Cx195Dx2A6Ex3B7F = _mm512_permutexvar_epi32(_mm512_set_epi32(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0), vout0122x084Cx195Dx2A6Ex3B7F);
- const __m512i vout0122x0123456789ABCDEF = _mm512_shuffle_epi8(vout0122x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ __m512i vout0122x0123456789ABCDEF = _mm512_shuffle_epi8(vout0122x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ vout0122x0123456789ABCDEF = _mm512_max_epi8(vout0122x0123456789ABCDEF, voutput_min);
+ vout0122x0123456789ABCDEF = _mm512_min_epi8(vout0122x0123456789ABCDEF, voutput_max);
if (nc >= 16) {
_mm_storeu_si128((__m128i*) c0, _mm512_castsi512_si128(vout0122x0123456789ABCDEF));
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-avx-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-avx-ld128.c
index 99c9140..06a9192 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-avx-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-avx-ld128.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -185,7 +185,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -193,8 +193,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -202,17 +202,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-avx-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-avx-ld64.c
index db03795..cff57f6 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-avx-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-avx-ld64.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -185,7 +185,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -193,8 +193,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -202,17 +202,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-sse2-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-sse2-ld128.c
index 026b9e2..a3e6c9f 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-sse2-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-sse2-ld128.c
@@ -226,7 +226,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -245,6 +245,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-sse2-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-sse2-ld64.c
index c5d590f..a6a69b4 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-sse2-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-sse2-ld64.c
@@ -226,7 +226,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -245,6 +245,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-sse41-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-sse41-ld128.c
index 79e7aeb..396470b 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-sse41-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-sse41-ld128.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -185,7 +185,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -193,8 +193,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -202,17 +202,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-sse41-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-sse41-ld64.c
index 63d8d88..227d609 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-sse41-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-sse41-ld64.c
@@ -159,8 +159,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -185,7 +185,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -193,8 +193,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -202,17 +202,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-ssse3-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-ssse3-ld128.c
index 2d2cd3e..306b6e5 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-ssse3-ld128.c
@@ -226,7 +226,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -245,6 +245,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-ssse3-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-ssse3-ld64.c
index 171dd4e..664fd04 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-ssse3-ld64.c
@@ -226,7 +226,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -245,6 +245,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-xop-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-xop-ld128.c
index 3ee5466..f599282 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-xop-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-xop-ld128.c
@@ -164,8 +164,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -190,7 +190,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -198,8 +198,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -207,17 +207,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-xop-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-xop-ld64.c
index 1281aa0..c7a4ceb 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-xop-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-xop-ld64.c
@@ -164,8 +164,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -190,7 +190,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -198,8 +198,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -207,17 +207,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-avx.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-avx.c
index f9b6311..0f5d9b1 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-avx.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-avx.c
@@ -152,8 +152,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -178,7 +178,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -186,8 +186,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -195,17 +195,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-sse2.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-sse2.c
index 51dd9bf..0b0c916 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-sse2.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-sse2.c
@@ -219,7 +219,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -238,6 +238,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-sse41.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-sse41.c
index b88c263..c4b2953 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-sse41.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-sse41.c
@@ -152,8 +152,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -178,7 +178,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -186,8 +186,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -195,17 +195,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-ssse3.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-ssse3.c
index 2806362..4cd3e0d 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-ssse3.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-ssse3.c
@@ -219,7 +219,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -238,6 +238,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-xop.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-xop.c
index c72ebdb..86d4b3f 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-xop.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-xop.c
@@ -157,8 +157,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -183,7 +183,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -191,8 +191,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -200,17 +200,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-avx-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-avx-ld128.c
index abcedde..a2bf9d9 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-avx-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-avx-ld128.c
@@ -117,8 +117,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -143,7 +143,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -151,8 +151,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -160,17 +160,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-avx-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-avx-ld64.c
index a92f659..72b3074 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-avx-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-avx-ld64.c
@@ -119,8 +119,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -145,7 +145,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -153,8 +153,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -162,17 +162,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-sse2-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-sse2-ld128.c
index 9bc9881..0f57e0f 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-sse2-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-sse2-ld128.c
@@ -184,7 +184,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -203,6 +203,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-sse2-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-sse2-ld64.c
index 7db069f..10fa7cd 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-sse2-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-sse2-ld64.c
@@ -186,7 +186,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -205,6 +205,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-sse41-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-sse41-ld128.c
index 5d1bf8f..6d8579c 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-sse41-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-sse41-ld128.c
@@ -117,8 +117,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -143,7 +143,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -151,8 +151,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -160,17 +160,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-sse41-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-sse41-ld64.c
index 7866238..62b0ba2 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-sse41-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-sse41-ld64.c
@@ -119,8 +119,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -145,7 +145,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -153,8 +153,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -162,17 +162,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-ssse3-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-ssse3-ld128.c
index 689ea6f..81d0de0 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-ssse3-ld128.c
@@ -184,7 +184,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -203,6 +203,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-ssse3-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-ssse3-ld64.c
index b545041..fa0121b 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-ssse3-ld64.c
@@ -186,7 +186,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -205,6 +205,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-xop-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-xop-ld128.c
index f1dd230..4a09afe 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-xop-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-xop-ld128.c
@@ -122,8 +122,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -148,7 +148,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -156,8 +156,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -165,17 +165,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-xop-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-xop-ld64.c
index 9f91256..0c874fc 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-xop-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-xop-ld64.c
@@ -124,8 +124,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -150,7 +150,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -158,8 +158,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -167,17 +167,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-avx.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-avx.c
index 8238c2d..a4192f6 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-avx.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-avx.c
@@ -115,8 +115,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -141,7 +141,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -149,8 +149,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -158,17 +158,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-sse2.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-sse2.c
index 51d77de..4d4d116 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-sse2.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-sse2.c
@@ -182,7 +182,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -201,6 +201,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-sse41.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-sse41.c
index ac20132..a00a4ca 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-sse41.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-sse41.c
@@ -115,8 +115,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -141,7 +141,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -149,8 +149,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -158,17 +158,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-ssse3.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-ssse3.c
index 80eae05..13681e3 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-ssse3.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-ssse3.c
@@ -182,7 +182,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -201,6 +201,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-xop.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-xop.c
index c8fa729..e471b8e 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-xop.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-xop.c
@@ -120,8 +120,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -146,7 +146,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -154,8 +154,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -163,17 +163,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/3x8c8-minmax-avx2.c b/src/qs8-gemm/gen/3x8c8-minmax-avx2.c
index fd0ce85..d41e047 100644
--- a/src/qs8-gemm/gen/3x8c8-minmax-avx2.c
+++ b/src/qs8-gemm/gen/3x8c8-minmax-avx2.c
@@ -133,12 +133,12 @@
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
__m256i vacc2x01234567 = _mm256_permutevar8x32_epi32(vacc2x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
- const __m256i vacc0x11335577 = _mm256_shuffle_epi32(vacc0x01234567, _MM_SHUFFLE(3, 3, 1, 1));
- const __m256i vacc1x11335577 = _mm256_shuffle_epi32(vacc1x01234567, _MM_SHUFFLE(3, 3, 1, 1));
- const __m256i vacc2x11335577 = _mm256_shuffle_epi32(vacc2x01234567, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
+ const __m256i vacc1x11335577 = _mm256_srli_epi64(vacc1x01234567, 32);
+ const __m256i vacc2x11335577 = _mm256_srli_epi64(vacc2x01234567, 32);
const __m256i vprod0x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc0x01234567, vmultiplier), vrounding);
const __m256i vprod1x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc1x01234567, vmultiplier), vrounding);
@@ -159,7 +159,7 @@
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
const __m256i vq31prod2x01234567 = _mm256_blend_epi16(vq31prod2x0246, vq31prod2x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
@@ -167,8 +167,8 @@
const __m256i vrem2x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod2x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod2x01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
@@ -176,19 +176,18 @@
vacc2x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod2x01234567, vshift), _mm256_cmpgt_epi32(vrem2x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
__m256i vacc22x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc2x01234567, vacc2x01234567), voutput_zero_point);
vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
vacc22x01234567 = _mm256_permute4x64_epi64(vacc22x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vacc01x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc01x01234567, voutput_min), voutput_max);
- vacc22x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc22x01234567, voutput_min), voutput_max);
-
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc22x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/3x8c8-xw-minmax-avx2.c b/src/qs8-gemm/gen/3x8c8-xw-minmax-avx2.c
index a84d8c4..d589e3c 100644
--- a/src/qs8-gemm/gen/3x8c8-xw-minmax-avx2.c
+++ b/src/qs8-gemm/gen/3x8c8-xw-minmax-avx2.c
@@ -129,12 +129,12 @@
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
__m256i vacc2x01234567 = _mm256_permutevar8x32_epi32(vacc2x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
- const __m256i vacc0x11335577 = _mm256_shuffle_epi32(vacc0x01234567, _MM_SHUFFLE(3, 3, 1, 1));
- const __m256i vacc1x11335577 = _mm256_shuffle_epi32(vacc1x01234567, _MM_SHUFFLE(3, 3, 1, 1));
- const __m256i vacc2x11335577 = _mm256_shuffle_epi32(vacc2x01234567, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m256i vacc0x11335577 = _mm256_srli_epi64(vacc0x01234567, 32);
+ const __m256i vacc1x11335577 = _mm256_srli_epi64(vacc1x01234567, 32);
+ const __m256i vacc2x11335577 = _mm256_srli_epi64(vacc2x01234567, 32);
const __m256i vprod0x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc0x01234567, vmultiplier), vrounding);
const __m256i vprod1x0246 = _mm256_add_epi64(_mm256_mul_epi32(vacc1x01234567, vmultiplier), vrounding);
@@ -155,7 +155,7 @@
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
const __m256i vq31prod2x01234567 = _mm256_blend_epi16(vq31prod2x0246, vq31prod2x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
@@ -163,8 +163,8 @@
const __m256i vrem2x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod2x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod2x01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
@@ -172,19 +172,18 @@
vacc2x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod2x01234567, vshift), _mm256_cmpgt_epi32(vrem2x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
__m256i vacc22x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc2x01234567, vacc2x01234567), voutput_zero_point);
vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
vacc22x01234567 = _mm256_permute4x64_epi64(vacc22x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vacc01x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc01x01234567, voutput_min), voutput_max);
- vacc22x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc22x01234567, voutput_min), voutput_max);
-
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc22x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-gemm/gen/4x16c8-minmax-avx512skx.c b/src/qs8-gemm/gen/4x16c8-minmax-avx512skx.c
index ccc7f19..2a7b6ca 100644
--- a/src/qs8-gemm/gen/4x16c8-minmax-avx512skx.c
+++ b/src/qs8-gemm/gen/4x16c8-minmax-avx512skx.c
@@ -61,14 +61,14 @@
const __mmask16 vbias_mask = _cvtu32_mask16(0x1111);
const __mmask16 vblend_mask = _cvtu32_mask16(0xAAAA);
- const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.rounding));
- const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
- const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i vmultiplier = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.multiplier));
+ const __m512i vrounding = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.rounding));
+ const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_mask));
+ const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.remainder_threshold));
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
@@ -195,15 +195,14 @@
vacc3x084C195D2A6E3B7F =
_mm512_mask_sub_epi32(vacc3x084C195D2A6E3B7F, _mm512_cmpgt_epi32_mask(vrem3x084C195D2A6E3B7F, vremainder_threshold), vacc3x084C195D2A6E3B7F, vminus_one);
- __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
- __m512i vacc23x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc2x084C195D2A6E3B7F, vacc3x084C195D2A6E3B7F), voutput_zero_point);
-
- vacc01x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc01x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
- vacc23x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc23x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
+ const __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
+ const __m512i vacc23x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc2x084C195D2A6E3B7F, vacc3x084C195D2A6E3B7F), voutput_zero_point);
__m512i vout0123x084Cx195Dx2A6Ex3B7F = _mm512_packs_epi16(vacc01x084Cx195Dx2A6Ex3B7F, vacc23x084Cx195Dx2A6Ex3B7F);
vout0123x084Cx195Dx2A6Ex3B7F = _mm512_permutexvar_epi32(_mm512_set_epi32(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0), vout0123x084Cx195Dx2A6Ex3B7F);
- const __m512i vout0123x0123456789ABCDEF = _mm512_shuffle_epi8(vout0123x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ __m512i vout0123x0123456789ABCDEF = _mm512_shuffle_epi8(vout0123x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ vout0123x0123456789ABCDEF = _mm512_max_epi8(vout0123x0123456789ABCDEF, voutput_min);
+ vout0123x0123456789ABCDEF = _mm512_min_epi8(vout0123x0123456789ABCDEF, voutput_max);
if (nc >= 16) {
_mm_storeu_si128((__m128i*) c0, _mm512_castsi512_si128(vout0123x0123456789ABCDEF));
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-avx-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-avx-ld128.c
index 69f3237..d3c0a28 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-avx-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-avx-ld128.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -218,7 +218,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -228,8 +228,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -239,17 +239,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-avx-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-avx-ld64.c
index 48cec4a..43fdc24 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-avx-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-avx-ld64.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -218,7 +218,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -228,8 +228,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -239,17 +239,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-sse2-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-sse2-ld128.c
index b11f3b5..20a69e6 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-sse2-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-sse2-ld128.c
@@ -269,7 +269,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -290,6 +290,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-sse2-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-sse2-ld64.c
index 57d930d..aed4404 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-sse2-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-sse2-ld64.c
@@ -269,7 +269,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -290,6 +290,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-sse41-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-sse41-ld128.c
index f45eed0..31a3cda 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-sse41-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-sse41-ld128.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -218,7 +218,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -228,8 +228,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -239,17 +239,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-sse41-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-sse41-ld64.c
index 14330a5..b2cf9d6 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-sse41-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-sse41-ld64.c
@@ -186,8 +186,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -218,7 +218,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -228,8 +228,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -239,17 +239,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-ssse3-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-ssse3-ld128.c
index 462829e..c1a756e 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-ssse3-ld128.c
@@ -269,7 +269,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -290,6 +290,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-ssse3-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-ssse3-ld64.c
index 2789fe5..d1b058d 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-ssse3-ld64.c
@@ -269,7 +269,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -290,6 +290,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-xop-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-xop-ld128.c
index 63f3f41..b1b2b86 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-xop-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-xop-ld128.c
@@ -191,8 +191,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -223,7 +223,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -233,8 +233,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -244,17 +244,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-xop-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-xop-ld64.c
index 7c30ef9..1bf5524 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-xop-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-xop-ld64.c
@@ -191,8 +191,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -223,7 +223,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -233,8 +233,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -244,17 +244,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-avx.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-avx.c
index ae7c5a9..8179aeb 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-avx.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-avx.c
@@ -179,8 +179,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -211,7 +211,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -221,8 +221,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -232,17 +232,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-sse2.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-sse2.c
index 9f90a03..60593d2 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-sse2.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-sse2.c
@@ -262,7 +262,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -283,6 +283,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-sse41.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-sse41.c
index 383b1a5..ca5a399 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-sse41.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-sse41.c
@@ -179,8 +179,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -211,7 +211,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -221,8 +221,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -232,17 +232,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-ssse3.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-ssse3.c
index bdb5950..a2cd7d0 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-ssse3.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-ssse3.c
@@ -262,7 +262,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -283,6 +283,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
vout = _mm_srli_si128(vout, 4);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-xop.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-xop.c
index c80140a..699237d 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-xop.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-xop.c
@@ -184,8 +184,8 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -216,7 +216,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -226,8 +226,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -237,17 +237,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
diff --git a/src/qs8-igemm/MRx16c8-avx512skx.c.in b/src/qs8-igemm/MRx16c8-avx512skx.c.in
index 55595b7..4eddff2 100644
--- a/src/qs8-igemm/MRx16c8-avx512skx.c.in
+++ b/src/qs8-igemm/MRx16c8-avx512skx.c.in
@@ -64,13 +64,18 @@
const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
$if MR > 1:
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
$else:
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ $if MR > 2:
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ $elif MR == 2:
+ const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_min));
+ const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_max));
+ $else:
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
$for N in range(4, 16, 4):
@@ -155,26 +160,28 @@
_mm512_mask_sub_epi32(vacc${M}x084C195D2A6E3B7F, _mm512_cmpgt_epi32_mask(vrem${M}x084C195D2A6E3B7F, vremainder_threshold), vacc${M}x084C195D2A6E3B7F, vminus_one);
$if MR == 1:
- __m256i vacc0x084C2A6E195D3B7F = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc0x084C195D2A6E3B7F), _mm512_extracti32x8_epi32(vacc0x084C195D2A6E3B7F, 1)), voutput_zero_point);
- vacc0x084C2A6E195D3B7F = _mm256_min_epi16(_mm256_max_epi16(vacc0x084C2A6E195D3B7F, voutput_min), voutput_max);
+ const __m256i vacc0x084C2A6E195D3B7F = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc0x084C195D2A6E3B7F), _mm512_extracti32x8_epi32(vacc0x084C195D2A6E3B7F, 1)), voutput_zero_point);
$else:
$for M in range(0, MR, 2):
- __m512i vacc${M}${min(M+1, MR-1)}x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc${M}x084C195D2A6E3B7F, vacc${min(M+1, MR-1)}x084C195D2A6E3B7F), voutput_zero_point);
-
- $for M in range(0, MR, 2):
- vacc${M}${min(M+1, MR-1)}x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc${M}${min(M+1, MR-1)}x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
+ const __m512i vacc${M}${min(M+1, MR-1)}x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc${M}x084C195D2A6E3B7F, vacc${min(M+1, MR-1)}x084C195D2A6E3B7F), voutput_zero_point);
$if MR > 2:
__m512i vout012${min(3, MR-1)}x084Cx195Dx2A6Ex3B7F = _mm512_packs_epi16(vacc01x084Cx195Dx2A6Ex3B7F, vacc2${min(3, MR-1)}x084Cx195Dx2A6Ex3B7F);
vout012${min(M+3, MR-1)}x084Cx195Dx2A6Ex3B7F = _mm512_permutexvar_epi32(_mm512_set_epi32(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0), vout012${min(3, MR-1)}x084Cx195Dx2A6Ex3B7F);
- const __m512i vout012${min(3, MR-1)}x0123456789ABCDEF = _mm512_shuffle_epi8(vout012${min(3, MR-1)}x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ __m512i vout012${min(3, MR-1)}x0123456789ABCDEF = _mm512_shuffle_epi8(vout012${min(3, MR-1)}x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ vout012${min(3, MR-1)}x0123456789ABCDEF = _mm512_max_epi8(vout012${min(3, MR-1)}x0123456789ABCDEF, voutput_min);
+ vout012${min(3, MR-1)}x0123456789ABCDEF = _mm512_min_epi8(vout012${min(3, MR-1)}x0123456789ABCDEF, voutput_max);
$elif MR == 2:
const __m256i vout01x084Cx2A6Ex195Dx3B7F = _mm256_packs_epi16(_mm512_castsi512_si256(vacc01x084Cx195Dx2A6Ex3B7F), _mm512_extracti32x8_epi32(vacc01x084Cx195Dx2A6Ex3B7F, 1));
const __m256i vout01x084C2A6E195D3B7F = _mm256_permutevar8x32_epi32(vout01x084Cx2A6Ex195Dx3B7F, _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0));
- const __m256i vout01x0123456789ABCDEF = _mm256_shuffle_epi8(vout01x084C2A6E195D3B7F, _mm256_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0, 15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ __m256i vout01x0123456789ABCDEF = _mm256_shuffle_epi8(vout01x084C2A6E195D3B7F, _mm256_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0, 15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ vout01x0123456789ABCDEF = _mm256_max_epi8(vout01x0123456789ABCDEF, voutput_min);
+ vout01x0123456789ABCDEF = _mm256_min_epi8(vout01x0123456789ABCDEF, voutput_max);
$elif MR == 1:
const __m128i vout0x084C2A6E195D3B7F = _mm_packs_epi16(_mm256_castsi256_si128(vacc0x084C2A6E195D3B7F), _mm256_extracti128_si256(vacc0x084C2A6E195D3B7F, 1));
- const __m128i vout0x0123456789ABCDEF = _mm_shuffle_epi8(vout0x084C2A6E195D3B7F, _mm_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ __m128i vout0x0123456789ABCDEF = _mm_shuffle_epi8(vout0x084C2A6E195D3B7F, _mm_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ vout0x0123456789ABCDEF = _mm_max_epi8(vout0x0123456789ABCDEF, voutput_min);
+ vout0x0123456789ABCDEF = _mm_min_epi8(vout0x0123456789ABCDEF, voutput_max);
$if MR > 2:
if (nc >= 16) {
diff --git a/src/qs8-igemm/MRx4c2-sse.c.in b/src/qs8-igemm/MRx4c2-sse.c.in
index e3cc0df..ded0cde 100644
--- a/src/qs8-igemm/MRx4c2-sse.c.in
+++ b/src/qs8-igemm/MRx4c2-sse.c.in
@@ -25,6 +25,7 @@
#include <xnnpack/math.h>
+$PARAMS_STRUCT = "sse4" if SSE >= 4 else "sse2"
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
void xnn_qs8_igemm_minmax_ukernel_${MR}x4c2__${ISA}_${VARIANT.lower()}(
size_t mr,
@@ -192,8 +193,8 @@
p -= ${MR} * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
$if SSE == 4:
$for M in range(MR):
@@ -255,31 +256,39 @@
$for M in range(MR):
const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
$for M in range(MR):
const __m128i vrem${M}x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
$for M in range(MR):
vacc${M}x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
$for M in range(0, MR, 2):
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- $for M in range(0, MR, 2):
- vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
+ $if SSE < 4:
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ $for M in range(0, MR, 2):
+ vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
$if MR > 2:
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc${min(2, MR-1)}${min(3, MR-1)}x0123);
$else:
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
+ $if SSE == 4:
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
$for M in reversed(range(1, MR)):
$if SSE == 4:
diff --git a/src/qs8-igemm/MRx4c8-sse.c.in b/src/qs8-igemm/MRx4c8-sse.c.in
index 0b4b236..2357c61 100644
--- a/src/qs8-igemm/MRx4c8-sse.c.in
+++ b/src/qs8-igemm/MRx4c8-sse.c.in
@@ -25,6 +25,7 @@
#include <xnnpack/math.h>
+$PARAMS_STRUCT = "sse4" if SSE >= 4 else "sse2"
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
void xnn_qs8_igemm_minmax_ukernel_${MR}x4c8__${ISA}_${VARIANT.lower()}(
size_t mr,
@@ -150,8 +151,8 @@
$for M in range(MR):
__m128i vacc${M}x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc${M}x02, vacc${M}x13), _mm_unpackhi_epi32(vacc${M}x02, vacc${M}x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
$if SSE == 4:
$for M in range(MR):
@@ -213,31 +214,39 @@
$for M in range(MR):
const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
$for M in range(MR):
const __m128i vrem${M}x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
$for M in range(MR):
vacc${M}x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
$for M in range(0, MR, 2):
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- $for M in range(0, MR, 2):
- vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
+ $if SSE < 4:
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
+ $for M in range(0, MR, 2):
+ vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
$if MR > 2:
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc${min(2, MR-1)}${min(3, MR-1)}x0123);
$else:
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
+ $if SSE == 4:
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
$for M in reversed(range(1, MR)):
$if SSE == 4:
diff --git a/src/qs8-igemm/MRx8c8-avx2.c.in b/src/qs8-igemm/MRx8c8-avx2.c.in
index 582f70c..34df8bb 100644
--- a/src/qs8-igemm/MRx8c8-avx2.c.in
+++ b/src/qs8-igemm/MRx8c8-avx2.c.in
@@ -111,8 +111,8 @@
$for M in range(MR):
__m256i vacc${M}x01234567 = _mm256_permutevar8x32_epi32(vacc${M}x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
$for M in range(MR):
const __m256i vacc${M}x11335577 = _mm256_shuffle_epi32(vacc${M}x01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -130,33 +130,35 @@
$for M in range(MR):
const __m256i vq31prod${M}x01234567 = _mm256_blend_epi16(vq31prod${M}x0246, vq31prod${M}x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
$for M in range(MR):
const __m256i vrem${M}x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod${M}x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod${M}x01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
$for M in range(MR):
vacc${M}x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod${M}x01234567, vshift), _mm256_cmpgt_epi32(vrem${M}x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
$for M in range(0, MR, 2):
__m256i vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc${M}x01234567, vacc${min(M+1, MR-1)}x01234567), voutput_zero_point);
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_permute4x64_epi64(vacc${M}${min(M+1, MR-1)}x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- $for M in range(0, MR, 2):
- vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc${M}${min(M+1, MR-1)}x01234567, voutput_min), voutput_max);
-
$if MR > 2:
__m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc${min(2, MR-1)}${min(3, MR-1)}x01234567);
$else:
__m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-igemm/gen/1x16c8-minmax-avx512skx.c b/src/qs8-igemm/gen/1x16c8-minmax-avx512skx.c
index 6e860c2..fd29a93 100644
--- a/src/qs8-igemm/gen/1x16c8-minmax-avx512skx.c
+++ b/src/qs8-igemm/gen/1x16c8-minmax-avx512skx.c
@@ -49,9 +49,9 @@
const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse4.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse4.output_max);
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
@@ -116,11 +116,12 @@
vacc0x084C195D2A6E3B7F =
_mm512_mask_sub_epi32(vacc0x084C195D2A6E3B7F, _mm512_cmpgt_epi32_mask(vrem0x084C195D2A6E3B7F, vremainder_threshold), vacc0x084C195D2A6E3B7F, vminus_one);
- __m256i vacc0x084C2A6E195D3B7F = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc0x084C195D2A6E3B7F), _mm512_extracti32x8_epi32(vacc0x084C195D2A6E3B7F, 1)), voutput_zero_point);
- vacc0x084C2A6E195D3B7F = _mm256_min_epi16(_mm256_max_epi16(vacc0x084C2A6E195D3B7F, voutput_min), voutput_max);
+ const __m256i vacc0x084C2A6E195D3B7F = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc0x084C195D2A6E3B7F), _mm512_extracti32x8_epi32(vacc0x084C195D2A6E3B7F, 1)), voutput_zero_point);
const __m128i vout0x084C2A6E195D3B7F = _mm_packs_epi16(_mm256_castsi256_si128(vacc0x084C2A6E195D3B7F), _mm256_extracti128_si256(vacc0x084C2A6E195D3B7F, 1));
- const __m128i vout0x0123456789ABCDEF = _mm_shuffle_epi8(vout0x084C2A6E195D3B7F, _mm_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ __m128i vout0x0123456789ABCDEF = _mm_shuffle_epi8(vout0x084C2A6E195D3B7F, _mm_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ vout0x0123456789ABCDEF = _mm_max_epi8(vout0x0123456789ABCDEF, voutput_min);
+ vout0x0123456789ABCDEF = _mm_min_epi8(vout0x0123456789ABCDEF, voutput_max);
if (nc >= 16) {
_mm_storeu_si128((__m128i*) c0, vout0x0123456789ABCDEF);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-avx-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-avx-ld128.c
index 2443e03..f309d27 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-avx-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-avx-ld128.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,24 +132,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-avx-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-avx-ld64.c
index b846272..852be0e 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-avx-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-avx-ld64.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,24 +132,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld128.c
index 8bcb718..74c720c 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld128.c
@@ -166,6 +166,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld64.c
index 8909f0d..12f9ea0 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-sse2-ld64.c
@@ -166,6 +166,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld128.c
index dc9dbeb..f3a9c3a 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld128.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,24 +132,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld64.c
index cc8372c..4e9e5f9 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-sse41-ld64.c
@@ -118,8 +118,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,24 +132,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld128.c
index 5f22e7c..4380fab 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld128.c
@@ -166,6 +166,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld64.c
index 7c923f1..655c345 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-ssse3-ld64.c
@@ -166,6 +166,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-xop-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-xop-ld128.c
index 37cbccf..95a2775 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-xop-ld128.c
@@ -123,8 +123,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -137,24 +137,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-xop-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-xop-ld64.c
index 81f421d..841e214 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-xop-ld64.c
@@ -123,8 +123,8 @@
p -= 1 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -137,24 +137,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-avx-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-avx-ld128.c
index 2d2d988..58d01b6 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-avx-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-avx-ld128.c
@@ -90,8 +90,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -104,24 +104,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-avx-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-avx-ld64.c
index 9283765..d2dd638 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-avx-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-avx-ld64.c
@@ -92,8 +92,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -106,24 +106,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld128.c
index 1b225a1..1793705 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld128.c
@@ -138,6 +138,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld64.c
index 0700e32..5d1fdae 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-sse2-ld64.c
@@ -140,6 +140,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld128.c
index 0ba0ed2..eaffaa6 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld128.c
@@ -90,8 +90,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -104,24 +104,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld64.c
index 5f96910..ad2344f 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-sse41-ld64.c
@@ -92,8 +92,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -106,24 +106,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld128.c
index 0c0454f..da9a1c1 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld128.c
@@ -138,6 +138,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld64.c
index 5de63b6..520ba62 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-ssse3-ld64.c
@@ -140,6 +140,7 @@
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-xop-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-xop-ld128.c
index 1c95c42..b0646d5 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-xop-ld128.c
@@ -95,8 +95,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -109,24 +109,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-xop-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-xop-ld64.c
index aa6bbd1..7403cf2 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-xop-ld64.c
@@ -97,8 +97,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -111,24 +111,24 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
diff --git a/src/qs8-igemm/gen/1x8c8-minmax-avx2.c b/src/qs8-igemm/gen/1x8c8-minmax-avx2.c
index 4372a66..b988ae6 100644
--- a/src/qs8-igemm/gen/1x8c8-minmax-avx2.c
+++ b/src/qs8-igemm/gen/1x8c8-minmax-avx2.c
@@ -104,8 +104,8 @@
const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc0x11335577 = _mm256_shuffle_epi32(vacc0x01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -118,25 +118,25 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vacc00x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc0x01234567), voutput_zero_point);
vacc00x01234567 = _mm256_permute4x64_epi64(vacc00x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vacc00x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc00x01234567, voutput_min), voutput_max);
-
__m256i vout = _mm256_packs_epi16(vacc00x01234567, vacc00x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-igemm/gen/2x16c8-minmax-avx512skx.c b/src/qs8-igemm/gen/2x16c8-minmax-avx512skx.c
index af495ea..1a3b682 100644
--- a/src/qs8-igemm/gen/2x16c8-minmax-avx512skx.c
+++ b/src/qs8-igemm/gen/2x16c8-minmax-avx512skx.c
@@ -53,9 +53,9 @@
const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_min));
+ const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
@@ -148,13 +148,13 @@
vacc1x084C195D2A6E3B7F =
_mm512_mask_sub_epi32(vacc1x084C195D2A6E3B7F, _mm512_cmpgt_epi32_mask(vrem1x084C195D2A6E3B7F, vremainder_threshold), vacc1x084C195D2A6E3B7F, vminus_one);
- __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
-
- vacc01x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc01x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
+ const __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
const __m256i vout01x084Cx2A6Ex195Dx3B7F = _mm256_packs_epi16(_mm512_castsi512_si256(vacc01x084Cx195Dx2A6Ex3B7F), _mm512_extracti32x8_epi32(vacc01x084Cx195Dx2A6Ex3B7F, 1));
const __m256i vout01x084C2A6E195D3B7F = _mm256_permutevar8x32_epi32(vout01x084Cx2A6Ex195Dx3B7F, _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0));
- const __m256i vout01x0123456789ABCDEF = _mm256_shuffle_epi8(vout01x084C2A6E195D3B7F, _mm256_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0, 15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ __m256i vout01x0123456789ABCDEF = _mm256_shuffle_epi8(vout01x084C2A6E195D3B7F, _mm256_set_epi8(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0, 15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0));
+ vout01x0123456789ABCDEF = _mm256_max_epi8(vout01x0123456789ABCDEF, voutput_min);
+ vout01x0123456789ABCDEF = _mm256_min_epi8(vout01x0123456789ABCDEF, voutput_max);
if (nc >= 16) {
_mm_storeu_si128((__m128i*) c1, _mm256_extracti128_si256(vout01x0123456789ABCDEF, 1));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-avx-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-avx-ld128.c
index 6c2f64a..cc08b92 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-avx-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-avx-ld128.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -167,28 +167,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-avx-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-avx-ld64.c
index 92c3ee4..21823a3 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-avx-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-avx-ld64.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -167,28 +167,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld128.c
index 2ae8eba..89f3698 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld128.c
@@ -213,6 +213,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld64.c
index c1ccebd..d59bf28 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-sse2-ld64.c
@@ -213,6 +213,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld128.c
index 10a96a7..b852014 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld128.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -167,28 +167,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld64.c
index b50953e..4532c35 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-sse41-ld64.c
@@ -147,8 +147,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -167,28 +167,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld128.c
index 0b72444..10fc1e4 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld128.c
@@ -213,6 +213,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld64.c
index 29c6aeb..175955b 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-ssse3-ld64.c
@@ -213,6 +213,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-xop-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-xop-ld128.c
index 4cbf968..71b9cab 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-xop-ld128.c
@@ -152,8 +152,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -172,28 +172,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-xop-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-xop-ld64.c
index 2f53629..88649df 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-xop-ld64.c
@@ -152,8 +152,8 @@
p -= 2 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -172,28 +172,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-avx-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-avx-ld128.c
index 05fed74..9307dc8 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-avx-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-avx-ld128.c
@@ -112,8 +112,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,28 +132,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-avx-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-avx-ld64.c
index 774b901..b92f417 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-avx-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-avx-ld64.c
@@ -114,8 +114,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -134,28 +134,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld128.c
index 6444d15..9304588 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld128.c
@@ -178,6 +178,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld64.c
index 5f948d5..c6773a8 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-sse2-ld64.c
@@ -180,6 +180,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld128.c
index daaacf7..0c1f2ab 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld128.c
@@ -112,8 +112,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -132,28 +132,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld64.c
index 4ad6652..96af574 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-sse41-ld64.c
@@ -114,8 +114,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -134,28 +134,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld128.c
index 99bb004..95a3f59 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld128.c
@@ -178,6 +178,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld64.c
index 971df2a..ec11e48 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-ssse3-ld64.c
@@ -180,6 +180,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-xop-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-xop-ld128.c
index ed3cc5c..bd3c5a7 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-xop-ld128.c
@@ -117,8 +117,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -137,28 +137,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-xop-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-xop-ld64.c
index 2032a31..bb0a7c2 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-xop-ld64.c
@@ -119,8 +119,8 @@
__m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -139,28 +139,28 @@
const __m128i vq31prod0x0123 = _mm_blend_epi16(vq31prod0x02, vq31prod0x13, 0xCC);
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod1x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod1x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod1x0123, vshift), _mm_cmpgt_epi32(vrem1x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
diff --git a/src/qs8-igemm/gen/2x8c8-minmax-avx2.c b/src/qs8-igemm/gen/2x8c8-minmax-avx2.c
index a82458a..b33bfa9 100644
--- a/src/qs8-igemm/gen/2x8c8-minmax-avx2.c
+++ b/src/qs8-igemm/gen/2x8c8-minmax-avx2.c
@@ -127,8 +127,8 @@
__m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc0x11335577 = _mm256_shuffle_epi32(vacc0x01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc1x11335577 = _mm256_shuffle_epi32(vacc1x01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -147,29 +147,29 @@
const __m256i vq31prod0x01234567 = _mm256_blend_epi16(vq31prod0x0246, vq31prod0x1357, 0xCC);
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod1x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod1x01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod1x01234567, vshift), _mm256_cmpgt_epi32(vrem1x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vacc01x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc01x01234567, voutput_min), voutput_max);
-
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc01x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-igemm/gen/3x16c8-minmax-avx512skx.c b/src/qs8-igemm/gen/3x16c8-minmax-avx512skx.c
index 722d4f3..73d32f0 100644
--- a/src/qs8-igemm/gen/3x16c8-minmax-avx512skx.c
+++ b/src/qs8-igemm/gen/3x16c8-minmax-avx512skx.c
@@ -57,9 +57,9 @@
const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
@@ -180,15 +180,14 @@
vacc2x084C195D2A6E3B7F =
_mm512_mask_sub_epi32(vacc2x084C195D2A6E3B7F, _mm512_cmpgt_epi32_mask(vrem2x084C195D2A6E3B7F, vremainder_threshold), vacc2x084C195D2A6E3B7F, vminus_one);
- __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
- __m512i vacc22x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc2x084C195D2A6E3B7F, vacc2x084C195D2A6E3B7F), voutput_zero_point);
-
- vacc01x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc01x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
- vacc22x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc22x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
+ const __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
+ const __m512i vacc22x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc2x084C195D2A6E3B7F, vacc2x084C195D2A6E3B7F), voutput_zero_point);
__m512i vout0122x084Cx195Dx2A6Ex3B7F = _mm512_packs_epi16(vacc01x084Cx195Dx2A6Ex3B7F, vacc22x084Cx195Dx2A6Ex3B7F);
vout0122x084Cx195Dx2A6Ex3B7F = _mm512_permutexvar_epi32(_mm512_set_epi32(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0), vout0122x084Cx195Dx2A6Ex3B7F);
- const __m512i vout0122x0123456789ABCDEF = _mm512_shuffle_epi8(vout0122x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ __m512i vout0122x0123456789ABCDEF = _mm512_shuffle_epi8(vout0122x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ vout0122x0123456789ABCDEF = _mm512_max_epi8(vout0122x0123456789ABCDEF, voutput_min);
+ vout0122x0123456789ABCDEF = _mm512_min_epi8(vout0122x0123456789ABCDEF, voutput_max);
if (nc >= 16) {
_mm_storeu_si128((__m128i*) c2, _mm512_extracti32x4_epi32(vout0122x0123456789ABCDEF, 2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-avx-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-avx-ld128.c
index 94a8a7c..07170b4 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-avx-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-avx-ld128.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -202,7 +202,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -210,8 +210,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -219,17 +219,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-avx-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-avx-ld64.c
index 2f1a22d..a6224db 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-avx-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-avx-ld64.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -202,7 +202,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -210,8 +210,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -219,17 +219,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld128.c
index 5933dd2..991e596 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld128.c
@@ -243,7 +243,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -262,6 +262,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld64.c
index 0e380ee..1dff2c6 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-sse2-ld64.c
@@ -243,7 +243,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -262,6 +262,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld128.c
index b441fae..e9ac047 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld128.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -202,7 +202,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -210,8 +210,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -219,17 +219,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld64.c
index 6223c65..cf64636 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-sse41-ld64.c
@@ -176,8 +176,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -202,7 +202,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -210,8 +210,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -219,17 +219,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld128.c
index 0b2a7bc..fedb5a0 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld128.c
@@ -243,7 +243,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -262,6 +262,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld64.c
index ab0c741..0a105fb 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-ssse3-ld64.c
@@ -243,7 +243,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -262,6 +262,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-xop-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-xop-ld128.c
index 1ffbfee..6fd187b 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-xop-ld128.c
@@ -181,8 +181,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -207,7 +207,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -215,8 +215,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -224,17 +224,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-xop-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-xop-ld64.c
index 30a0e81..c0a46f4 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-xop-ld64.c
@@ -181,8 +181,8 @@
p -= 3 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -207,7 +207,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -215,8 +215,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -224,17 +224,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-avx-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-avx-ld128.c
index ee09dd0..7d5fff1 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-avx-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-avx-ld128.c
@@ -134,8 +134,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -160,7 +160,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -168,8 +168,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -177,17 +177,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-avx-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-avx-ld64.c
index dae0701..330eaa0 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-avx-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-avx-ld64.c
@@ -136,8 +136,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -162,7 +162,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -170,8 +170,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -179,17 +179,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld128.c
index c0d9941..0022e00 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld128.c
@@ -201,7 +201,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -220,6 +220,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld64.c
index aaf3649..83d4334 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-sse2-ld64.c
@@ -203,7 +203,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -222,6 +222,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld128.c
index 92c755b..a39c3ab 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld128.c
@@ -134,8 +134,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -160,7 +160,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -168,8 +168,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -177,17 +177,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld64.c
index 069b8aa..82eec9e 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-sse41-ld64.c
@@ -136,8 +136,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -162,7 +162,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -170,8 +170,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -179,17 +179,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld128.c
index 69adc9b..0a35711 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld128.c
@@ -201,7 +201,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -220,6 +220,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld64.c
index 6afce4a..27e040c 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-ssse3-ld64.c
@@ -203,7 +203,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -222,6 +222,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-xop-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-xop-ld128.c
index 0ffb9cc..96d42c0 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-xop-ld128.c
@@ -139,8 +139,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -165,7 +165,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -173,8 +173,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -182,17 +182,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-xop-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-xop-ld64.c
index b7d59b6..f311e3c 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-xop-ld64.c
@@ -141,8 +141,8 @@
__m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
__m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -167,7 +167,7 @@
const __m128i vq31prod1x0123 = _mm_blend_epi16(vq31prod1x02, vq31prod1x13, 0xCC);
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -175,8 +175,8 @@
const __m128i vrem2x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod2x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod2x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -184,17 +184,16 @@
vacc2x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod2x0123, vshift), _mm_cmpgt_epi32(vrem2x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
diff --git a/src/qs8-igemm/gen/3x8c8-minmax-avx2.c b/src/qs8-igemm/gen/3x8c8-minmax-avx2.c
index 202bb77..2055125 100644
--- a/src/qs8-igemm/gen/3x8c8-minmax-avx2.c
+++ b/src/qs8-igemm/gen/3x8c8-minmax-avx2.c
@@ -150,8 +150,8 @@
__m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
__m256i vacc2x01234567 = _mm256_permutevar8x32_epi32(vacc2x02461357, vpermute_mask);
- const __m256i vmultiplier = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.multiplier));
- const __m256i vrounding = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.rounding));
+ const __m256i vmultiplier = _mm256_load_si256((const __m256i*) params->avx2.multiplier);
+ const __m256i vrounding = _mm256_load_si256((const __m256i*) params->avx2.rounding);
const __m256i vacc0x11335577 = _mm256_shuffle_epi32(vacc0x01234567, _MM_SHUFFLE(3, 3, 1, 1));
const __m256i vacc1x11335577 = _mm256_shuffle_epi32(vacc1x01234567, _MM_SHUFFLE(3, 3, 1, 1));
@@ -176,7 +176,7 @@
const __m256i vq31prod1x01234567 = _mm256_blend_epi16(vq31prod1x0246, vq31prod1x1357, 0xCC);
const __m256i vq31prod2x01234567 = _mm256_blend_epi16(vq31prod2x0246, vq31prod2x1357, 0xCC);
- const __m256i vremainder_mask = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
+ const __m256i vremainder_mask = _mm256_load_si256((const __m256i*) params->avx2.remainder_mask);
const __m256i vrem0x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod0x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod0x01234567));
const __m256i vrem1x01234567 =
@@ -184,8 +184,8 @@
const __m256i vrem2x01234567 =
_mm256_add_epi32(_mm256_and_si256(vq31prod2x01234567, vremainder_mask), _mm256_cmpgt_epi32(_mm256_setzero_si256(), vq31prod2x01234567));
- const __m256i vremainder_threshold = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m256i vremainder_threshold = _mm256_load_si256((const __m256i*) params->avx2.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->avx2.shift);
vacc0x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod0x01234567, vshift), _mm256_cmpgt_epi32(vrem0x01234567, vremainder_threshold));
vacc1x01234567 =
@@ -193,19 +193,18 @@
vacc2x01234567 =
_mm256_sub_epi32(_mm256_sra_epi32(vq31prod2x01234567, vshift), _mm256_cmpgt_epi32(vrem2x01234567, vremainder_threshold));
- const __m256i voutput_zero_point = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx2.output_zero_point);
__m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
__m256i vacc22x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc2x01234567, vacc2x01234567), voutput_zero_point);
vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
vacc22x01234567 = _mm256_permute4x64_epi64(vacc22x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- const __m256i voutput_min = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m256i voutput_max = _mm256_broadcastsi128_si256(_mm_load_si128((const __m128i*) params->sse2.output_max));
- vacc01x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc01x01234567, voutput_min), voutput_max);
- vacc22x01234567 = _mm256_min_epi16(_mm256_max_epi16(vacc22x01234567, voutput_min), voutput_max);
-
__m256i vout = _mm256_packs_epi16(vacc01x01234567, vacc22x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->avx2.output_max));
+
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
diff --git a/src/qs8-igemm/gen/4x16c8-minmax-avx512skx.c b/src/qs8-igemm/gen/4x16c8-minmax-avx512skx.c
index 4828185..2752f79 100644
--- a/src/qs8-igemm/gen/4x16c8-minmax-avx512skx.c
+++ b/src/qs8-igemm/gen/4x16c8-minmax-avx512skx.c
@@ -61,9 +61,9 @@
const __m512i vremainder_mask = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_mask));
const __m512i vremainder_threshold = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.remainder_threshold));
const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
- const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_zero_point));
- const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_min));
- const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse2.output_max));
+ const __m512i voutput_zero_point = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_zero_point));
+ const __m512i voutput_min = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_min));
+ const __m512i voutput_max = _mm512_broadcast_i32x4(_mm_load_si128((const __m128i*) params->sse4.output_max));
do {
__m512i vacc0x0123 = _mm512_maskz_expandloadu_epi32(vbias_mask, w);
__m512i vacc0x4567 = _mm512_maskz_expandloadu_epi32(vbias_mask, (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
@@ -212,15 +212,14 @@
vacc3x084C195D2A6E3B7F =
_mm512_mask_sub_epi32(vacc3x084C195D2A6E3B7F, _mm512_cmpgt_epi32_mask(vrem3x084C195D2A6E3B7F, vremainder_threshold), vacc3x084C195D2A6E3B7F, vminus_one);
- __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
- __m512i vacc23x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc2x084C195D2A6E3B7F, vacc3x084C195D2A6E3B7F), voutput_zero_point);
-
- vacc01x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc01x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
- vacc23x084Cx195Dx2A6Ex3B7F = _mm512_min_epi16(_mm512_max_epi16(vacc23x084Cx195Dx2A6Ex3B7F, voutput_min), voutput_max);
+ const __m512i vacc01x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc0x084C195D2A6E3B7F, vacc1x084C195D2A6E3B7F), voutput_zero_point);
+ const __m512i vacc23x084Cx195Dx2A6Ex3B7F = _mm512_adds_epi16(_mm512_packs_epi32(vacc2x084C195D2A6E3B7F, vacc3x084C195D2A6E3B7F), voutput_zero_point);
__m512i vout0123x084Cx195Dx2A6Ex3B7F = _mm512_packs_epi16(vacc01x084Cx195Dx2A6Ex3B7F, vacc23x084Cx195Dx2A6Ex3B7F);
vout0123x084Cx195Dx2A6Ex3B7F = _mm512_permutexvar_epi32(_mm512_set_epi32(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0), vout0123x084Cx195Dx2A6Ex3B7F);
- const __m512i vout0123x0123456789ABCDEF = _mm512_shuffle_epi8(vout0123x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ __m512i vout0123x0123456789ABCDEF = _mm512_shuffle_epi8(vout0123x084Cx195Dx2A6Ex3B7F, _mm512_set_epi8(15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0, 15, 11, 7, 3, 13, 9, 5, 1, 14, 10, 6, 2, 12, 8, 4, 0));
+ vout0123x0123456789ABCDEF = _mm512_max_epi8(vout0123x0123456789ABCDEF, voutput_min);
+ vout0123x0123456789ABCDEF = _mm512_min_epi8(vout0123x0123456789ABCDEF, voutput_max);
if (nc >= 16) {
_mm_storeu_si128((__m128i*) c3, _mm512_extracti32x4_epi32(vout0123x0123456789ABCDEF, 3));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-avx-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-avx-ld128.c
index 90e9f37..e341603 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-avx-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-avx-ld128.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -237,7 +237,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -247,8 +247,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -258,17 +258,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-avx-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-avx-ld64.c
index 64aa971..2d815c9 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-avx-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-avx-ld64.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -237,7 +237,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -247,8 +247,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -258,17 +258,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld128.c
index 725a7aa..3e86a97 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld128.c
@@ -288,7 +288,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -309,6 +309,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld64.c
index 636f501..bdb9caa 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-sse2-ld64.c
@@ -288,7 +288,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -309,6 +309,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld128.c
index 0d511f1..09e76c0 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld128.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -237,7 +237,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -247,8 +247,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -258,17 +258,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld64.c
index 894e46a..66c499c 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-sse41-ld64.c
@@ -205,8 +205,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -237,7 +237,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -247,8 +247,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -258,17 +258,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld128.c
index 2d1aba8..d8559fe 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld128.c
@@ -288,7 +288,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -309,6 +309,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld64.c
index 11ff0c6..2318f10 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-ssse3-ld64.c
@@ -288,7 +288,7 @@
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse2.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -309,6 +309,7 @@
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-xop-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-xop-ld128.c
index 16a0c0a..0ce5964 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-xop-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-xop-ld128.c
@@ -210,8 +210,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -242,7 +242,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -252,8 +252,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -263,17 +263,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-xop-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-xop-ld64.c
index b4b0e7e..b44fa57 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-xop-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-xop-ld64.c
@@ -210,8 +210,8 @@
p -= 4 * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse2.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse2.rounding);
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->sse4.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->sse4.rounding);
const __m128i vacc0x1133 = _mm_shuffle_epi32(vacc0x0123, _MM_SHUFFLE(3, 3, 1, 1));
const __m128i vacc1x1133 = _mm_shuffle_epi32(vacc1x0123, _MM_SHUFFLE(3, 3, 1, 1));
@@ -242,7 +242,7 @@
const __m128i vq31prod2x0123 = _mm_blend_epi16(vq31prod2x02, vq31prod2x13, 0xCC);
const __m128i vq31prod3x0123 = _mm_blend_epi16(vq31prod3x02, vq31prod3x13, 0xCC);
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse2.remainder_mask);
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->sse4.remainder_mask);
const __m128i vrem0x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod0x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod0x0123));
const __m128i vrem1x0123 =
@@ -252,8 +252,8 @@
const __m128i vrem3x0123 =
_mm_add_epi32(_mm_and_si128(vq31prod3x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod3x0123));
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse2.remainder_threshold);
- const __m128i vshift = _mm_load_si128((const __m128i*) params->sse2.shift);
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->sse4.remainder_threshold);
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->sse4.shift);
vacc0x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0x0123, vshift), _mm_cmpgt_epi32(vrem0x0123, vremainder_threshold));
vacc1x0123 =
@@ -263,17 +263,16 @@
vacc3x0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod3x0123, vshift), _mm_cmpgt_epi32(vrem3x0123, vremainder_threshold));
- const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
__m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.output_max);
- vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
- vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
__m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->sse4.output_max));
+
if (nc >= 4) {
*((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
diff --git a/src/xnnpack/params-init.h b/src/xnnpack/params-init.h
index d396625..137e859 100644
--- a/src/xnnpack/params-init.h
+++ b/src/xnnpack/params-init.h
@@ -224,6 +224,99 @@
params->sse2.output_max[i] = (int16_t) output_max;
}
}
+
+static inline void xnn_init_qs8_gemm_sse4_params(
+ union xnn_qs8_gemm_params params[XNN_MIN_ELEMENTS(1)],
+ float scale,
+ int8_t output_zero_point,
+ int8_t output_min,
+ int8_t output_max)
+{
+ // Compute requantization parameters.
+ const uint32_t scale_bits = fp32_to_bits(scale);
+
+ // Multiplier is in [0x40000000, 0x7FFFFF80] range.
+ const int32_t multiplier = (int32_t)(((scale_bits & UINT32_C(0x007FFFFF)) | UINT32_C(0x00800000)) << 7);
+ assert(multiplier >= INT32_C(0x40000000));
+ assert(multiplier <= INT32_C(0x7FFFFF80));
+
+ // Shift is in [0, 31] range.
+ const int32_t shift = 127 + 31 - 32 - (fp32_to_bits(scale) >> 23);
+ assert(shift >= 0);
+ assert(shift < 32);
+
+ const uint32_t remainder_mask = (UINT32_C(1) << shift) - UINT32_C(1);
+ const uint32_t remainder_threshold = remainder_mask >> 1;
+ params->sse4.multiplier[0] = multiplier;
+ params->sse4.multiplier[1] = multiplier;
+ params->sse4.multiplier[2] = multiplier;
+ params->sse4.multiplier[3] = multiplier;
+ params->sse4.rounding[0] = UINT64_C(0x40000000);
+ params->sse4.rounding[1] = UINT64_C(0x40000000);
+ params->sse4.remainder_mask[0] = (int32_t) remainder_mask;
+ params->sse4.remainder_mask[1] = (int32_t) remainder_mask;
+ params->sse4.remainder_mask[2] = (int32_t) remainder_mask;
+ params->sse4.remainder_mask[3] = (int32_t) remainder_mask;
+ params->sse4.remainder_threshold[0] = (int32_t) remainder_threshold;
+ params->sse4.remainder_threshold[1] = (int32_t) remainder_threshold;
+ params->sse4.remainder_threshold[2] = (int32_t) remainder_threshold;
+ params->sse4.remainder_threshold[3] = (int32_t) remainder_threshold;
+ params->sse4.shift[0] = (uint64_t) (uint32_t) shift;
+ params->sse4.shift[1] = (uint64_t) (uint32_t) shift;
+ for (uint32_t i = 0; i < 8; i++) {
+ params->sse4.output_zero_point[i] = (int16_t) output_zero_point;
+ }
+ for (uint32_t i = 0; i < 16; i++) {
+ params->sse4.output_min[i] = output_min;
+ params->sse4.output_max[i] = output_max;
+ }
+}
+
+static inline void xnn_init_qs8_gemm_avx2_params(
+ union xnn_qs8_gemm_params params[XNN_MIN_ELEMENTS(1)],
+ float scale,
+ int8_t output_zero_point,
+ int8_t output_min,
+ int8_t output_max)
+{
+ // Compute requantization parameters.
+ const uint32_t scale_bits = fp32_to_bits(scale);
+
+ // Multiplier is in [0x40000000, 0x7FFFFF80] range.
+ const int32_t multiplier = (int32_t)(((scale_bits & UINT32_C(0x007FFFFF)) | UINT32_C(0x00800000)) << 7);
+ assert(multiplier >= INT32_C(0x40000000));
+ assert(multiplier <= INT32_C(0x7FFFFF80));
+
+ // Shift is in [0, 31] range.
+ const int32_t shift = 127 + 31 - 32 - (fp32_to_bits(scale) >> 23);
+ assert(shift >= 0);
+ assert(shift < 32);
+
+ const uint32_t remainder_mask = (UINT32_C(1) << shift) - UINT32_C(1);
+ const uint32_t remainder_threshold = remainder_mask >> 1;
+ for (uint32_t i = 0; i < 8; i++) {
+ params->avx2.multiplier[i] = multiplier;
+ }
+ params->avx2.rounding[0] = UINT64_C(0x40000000);
+ params->avx2.rounding[1] = UINT64_C(0x40000000);
+ params->avx2.rounding[2] = UINT64_C(0x40000000);
+ params->avx2.rounding[3] = UINT64_C(0x40000000);
+ for (uint32_t i = 0; i < 8; i++) {
+ params->avx2.remainder_mask[i] = (int32_t) remainder_mask;
+ params->avx2.remainder_threshold[i] = (int32_t) remainder_threshold;
+ }
+ params->avx2.shift[0] = (uint64_t) (uint32_t) shift;
+ params->avx2.shift[1] = (uint64_t) (uint32_t) shift;
+ params->avx2.shift[2] = (uint64_t) (uint32_t) shift;
+ params->avx2.shift[3] = (uint64_t) (uint32_t) shift;
+ for (uint32_t i = 0; i < 16; i++) {
+ params->avx2.output_zero_point[i] = (int16_t) output_zero_point;
+ }
+ for (uint32_t i = 0; i < 32; i++) {
+ params->avx2.output_min[i] = output_min;
+ params->avx2.output_max[i] = output_max;
+ }
+}
#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
#if XNN_ARCH_ARM || XNN_ARCH_ARM64
diff --git a/src/xnnpack/params.h b/src/xnnpack/params.h
index 66625c4..43a400b 100644
--- a/src/xnnpack/params.h
+++ b/src/xnnpack/params.h
@@ -308,6 +308,26 @@
XNN_ALIGN(16) int16_t output_min[8];
XNN_ALIGN(16) int16_t output_max[8];
} sse2;
+ struct {
+ XNN_ALIGN(16) uint32_t multiplier[4];
+ XNN_ALIGN(16) uint64_t rounding[2];
+ XNN_ALIGN(16) int32_t remainder_mask[4];
+ XNN_ALIGN(16) int32_t remainder_threshold[4];
+ XNN_ALIGN(16) uint64_t shift[2];
+ XNN_ALIGN(16) int16_t output_zero_point[8];
+ XNN_ALIGN(16) int8_t output_min[16];
+ XNN_ALIGN(16) int8_t output_max[16];
+ } sse4;
+ struct {
+ XNN_ALIGN(32) uint32_t multiplier[8];
+ XNN_ALIGN(32) uint64_t rounding[4];
+ XNN_ALIGN(32) int32_t remainder_mask[8];
+ XNN_ALIGN(32) int32_t remainder_threshold[8];
+ XNN_ALIGN(32) uint64_t shift[4];
+ XNN_ALIGN(32) int16_t output_zero_point[16];
+ XNN_ALIGN(32) int8_t output_min[32];
+ XNN_ALIGN(32) int8_t output_max[32];
+ } avx2;
#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
#if XNN_ARCH_WASMSIMD
struct {