FP32 requantization in QS8 GEMM/IGEMM microkernels for SSE/AVX/XOP
PiperOrigin-RevId: 376966195
diff --git a/src/init.c b/src/init.c
index 49ed439..fb353b3 100644
--- a/src/init.c
+++ b/src/init.c
@@ -1570,11 +1570,11 @@
xnn_params.qs8.gemm.log2_kr = 3;
} else if (cpuinfo_has_x86_xop()) {
// XOP should be checked before AVX2: AMD Excavator supports both, but performs better with XOP microkernels
- xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__xop_ld64);
- xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__xop_ld64);
- xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__xop_ld64);
- xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__xop_ld64);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
+ xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64);
+ xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld64);
+ xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64);
+ xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld64);
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse4_params;
xnn_params.qs8.gemm.mr = 2;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
@@ -1588,38 +1588,38 @@
xnn_params.qs8.gemm.nr = 8;
xnn_params.qs8.gemm.log2_kr = 3;
} else if (cpuinfo_has_x86_avx()) {
- xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__avx_ld128);
- xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__avx_ld128);
- xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__avx_ld128);
- xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__avx_ld128);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
+ xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128);
+ xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld128);
+ xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128);
+ xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld128);
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse4_params;
xnn_params.qs8.gemm.mr = 2;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
} else if (cpuinfo_has_x86_sse4_1()) {
- xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse41_ld64);
- xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__sse41_ld64);
- xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__sse41_ld64);
- xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__sse41_ld64);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse4_params;
+ xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64);
+ xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld64);
+ xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64);
+ xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld64);
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse4_params;
xnn_params.qs8.gemm.mr = 3;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
} else if (cpuinfo_has_x86_ssse3()) {
- xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__ssse3_ld64);
- xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__ssse3_ld64);
- xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__ssse3_ld64);
- xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__ssse3_ld64);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse2_params;
+ xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64);
+ xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64);
+ xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64);
+ xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64);
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse2_params;
xnn_params.qs8.gemm.mr = 3;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
} else {
- xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse2_ld64);
- xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__sse2_ld64);
- xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__sse2_ld64);
- xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__sse2_ld64);
- xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_gemmlowp_sse2_params;
+ xnn_params.qs8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64);
+ xnn_params.qs8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld64);
+ xnn_params.qs8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64);
+ xnn_params.qs8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld64);
+ xnn_params.qs8.gemm.init.qs8 = xnn_init_qs8_conv_minmax_fp32_sse2_params;
xnn_params.qs8.gemm.mr = 3;
xnn_params.qs8.gemm.nr = 4;
xnn_params.qs8.gemm.log2_kr = 3;
diff --git a/src/qs8-gemm/MRx4c2-sse.c.in b/src/qs8-gemm/MRx4c2-sse.c.in
index cdd7e71..6891ecf 100644
--- a/src/qs8-gemm/MRx4c2-sse.c.in
+++ b/src/qs8-gemm/MRx4c2-sse.c.in
@@ -6,6 +6,7 @@
$assert SSE in [2, 3, 4]
$assert not XOP or AVX
$assert not AVX or SSE == 4
+$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
$assert VARIANT in ["LD64", "LD128", "EXTENDED"]
$assert MR <= 4
#include <assert.h>
@@ -27,9 +28,9 @@
$LOAD_SUFFIX = {"LD128": "_ld128", "LD64": "_ld64", "EXTENDED": ""}[VARIANT]
$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else ""
-$PARAMS_STRUCT = "gemmlowp_sse4" if SSE >= 4 else "gemmlowp_sse2"
+$PARAMS_STRUCT = REQUANTIZATION.lower() + ("_sse4" if SSE >= 4 else "_sse2")
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
-void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_gemmlowp_ukernel_${MR}x4c2__${ISA}${LOAD_SUFFIX}(
+void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x4c2__${ISA}${LOAD_SUFFIX}(
size_t mr,
size_t nc,
size_t kc,
@@ -215,90 +216,101 @@
}
}
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
+ $if REQUANTIZATION == "GEMMLOWP":
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
- $if SSE == 4:
+ $if SSE == 4:
+ $for M in range(MR):
+ const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
+ const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
+ $else:
+ $for M in range(MR):
+ const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
+
+ $for M in range(MR):
+ $if SSE >= 3:
+ const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
+ $else:
+ const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
+
+ $for M in range(MR):
+ const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
+ _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
$for M in range(MR):
- const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m128i vrem${M}x0123 =
+ _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
+
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $for M in range(MR):
+ vacc${M}x0123 =
+ _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ $elif REQUANTIZATION == "FP32":
+ $for M in range(MR):
+ __m128 vscaled${M}x0123 = _mm_cvtepi32_ps(vacc${M}x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_${"sse4" if SSE >= 4 else "sse2"}.scale);
+ $for M in range(MR):
+ vscaled${M}x0123 = _mm_mul_ps(vscaled${M}x0123, vscale);
$for M in range(MR):
- const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
- const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
- $else:
- $for M in range(MR):
- const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
-
- $for M in range(MR):
- $if SSE >= 3:
- const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
- $else:
- const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
-
- $for M in range(MR):
- const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
-
- $for M in range(MR):
- const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
- _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
-
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
- $for M in range(MR):
- const __m128i vrem${M}x0123 =
- _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
-
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
- $if M > 1:
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $else:
- const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $for M in range(MR):
- vacc${M}x0123 =
- _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ vacc${M}x0123 = _mm_cvtps_epi32(vscaled${M}x0123);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
$for M in range(0, MR, 2):
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
$if SSE < 4:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_max);
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
@@ -308,8 +320,8 @@
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
$if SSE == 4:
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/MRx4c8-sse.c.in b/src/qs8-gemm/MRx4c8-sse.c.in
index 6f26c05..7c505d9 100644
--- a/src/qs8-gemm/MRx4c8-sse.c.in
+++ b/src/qs8-gemm/MRx4c8-sse.c.in
@@ -6,6 +6,7 @@
$assert SSE in [2, 3, 4]
$assert not XOP or AVX
$assert not AVX or SSE == 4
+$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
$assert VARIANT in ["LD64", "LD128", "EXTENDED"]
$assert MR <= 4
#include <assert.h>
@@ -27,9 +28,9 @@
$LOAD_SUFFIX = {"LD128": "_ld128", "LD64": "_ld64", "EXTENDED": ""}[VARIANT]
$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else ""
-$PARAMS_STRUCT = "gemmlowp_sse4" if SSE >= 4 else "gemmlowp_sse2"
+$PARAMS_STRUCT = REQUANTIZATION.lower() + ("_sse4" if SSE >= 4 else "_sse2")
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
-void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_gemmlowp_ukernel_${MR}x4c8__${ISA}${LOAD_SUFFIX}(
+void xnn_qs8_gemm${GEMM_SUFFIX}_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x4c8__${ISA}${LOAD_SUFFIX}(
size_t mr,
size_t nc,
size_t kc,
@@ -152,90 +153,101 @@
$for M in range(MR):
__m128i vacc${M}x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc${M}x02, vacc${M}x13), _mm_unpackhi_epi32(vacc${M}x02, vacc${M}x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
+ $if REQUANTIZATION == "GEMMLOWP":
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
- $if SSE == 4:
+ $if SSE == 4:
+ $for M in range(MR):
+ const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
+ const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
+ $else:
+ $for M in range(MR):
+ const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
+
+ $for M in range(MR):
+ $if SSE >= 3:
+ const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
+ $else:
+ const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
+
+ $for M in range(MR):
+ const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
+ _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
$for M in range(MR):
- const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m128i vrem${M}x0123 =
+ _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
+
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $for M in range(MR):
+ vacc${M}x0123 =
+ _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ $elif REQUANTIZATION == "FP32":
+ $for M in range(MR):
+ __m128 vscaled${M}x0123 = _mm_cvtepi32_ps(vacc${M}x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_${"sse4" if SSE >= 4 else "sse2"}.scale);
+ $for M in range(MR):
+ vscaled${M}x0123 = _mm_mul_ps(vscaled${M}x0123, vscale);
$for M in range(MR):
- const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
- const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
- $else:
- $for M in range(MR):
- const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
-
- $for M in range(MR):
- $if SSE >= 3:
- const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
- $else:
- const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
-
- $for M in range(MR):
- const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
-
- $for M in range(MR):
- const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
- _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
-
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
- $for M in range(MR):
- const __m128i vrem${M}x0123 =
- _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
-
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
- $if M > 1:
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $else:
- const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $for M in range(MR):
- vacc${M}x0123 =
- _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ vacc${M}x0123 = _mm_cvtps_epi32(vscaled${M}x0123);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
$for M in range(0, MR, 2):
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
$if SSE < 4:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_max);
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
@@ -245,8 +257,8 @@
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
$if SSE == 4:
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_max));
if (nc >= 4) {
*((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..841fdc5
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,145 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..ecc7bf0
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,145 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..202d416
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,146 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..3e8d08d
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,146 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..5ee38c5
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,145 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..4a61f66
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,145 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..145b151
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,146 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..6b207e9
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,146 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..c971ef9
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,150 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..cac9d9e
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,150 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..ed48fc0
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c
@@ -0,0 +1,117 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..da63466
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c
@@ -0,0 +1,119 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..da10e26
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,118 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..4b1bc7f
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,120 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..4009af4
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,117 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..7a806d0
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,119 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..ad24d67
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,118 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..641d3dd
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,120 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..5c20f88
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c
@@ -0,0 +1,122 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..9860e7b
--- /dev/null
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c
@@ -0,0 +1,124 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..c03b700
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,181 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..28c0f03
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,181 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..31f042a
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,183 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..cc1da18
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,183 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..dcbe48c
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,181 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..5b42ad3
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,181 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..055c4b3
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,183 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..90c0c93
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,183 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..bb8681d
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,186 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..a7b8771
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,186 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..791cbd4
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c
@@ -0,0 +1,146 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..ac2c116
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c
@@ -0,0 +1,148 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..3326d1e
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,148 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..5dc8d44
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,150 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..69ad9c2
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,146 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..3d02fa6
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,148 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..bff063f
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,148 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..43a1e38
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,150 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..eeeb805
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c
@@ -0,0 +1,151 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..53259fa
--- /dev/null
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c
@@ -0,0 +1,153 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..7bc2350
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,218 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..8d3eaa3
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,218 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..bf5d673
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,222 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..2040f86
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,222 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..77be744
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,218 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..8ccc871
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,218 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..26c0fd6
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,222 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..ef4a873
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,222 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..8d862f3
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,223 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..60cce10
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,223 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..167c725
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c
@@ -0,0 +1,176 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..1a54b46
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c
@@ -0,0 +1,178 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..d0c3877
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,180 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+ const __m128i vacc2x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x0, vacc2x2), _mm_unpackhi_epi32(vacc2x0, vacc2x2));
+ const __m128i vacc2x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x1, vacc2x3), _mm_unpackhi_epi32(vacc2x1, vacc2x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+ __m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..3bc4181
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,182 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+ const __m128i vacc2x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x0, vacc2x2), _mm_unpackhi_epi32(vacc2x0, vacc2x2));
+ const __m128i vacc2x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x1, vacc2x3), _mm_unpackhi_epi32(vacc2x1, vacc2x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+ __m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..dd4a934
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,176 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..6fe9768
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,178 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..0c042d1
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,180 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..6ae7371
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,182 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..4772e48
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c
@@ -0,0 +1,181 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
+ vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+ vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
+ vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..9dba8f5
--- /dev/null
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c
@@ -0,0 +1,183 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+ vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k += 8 * sizeof(int8_t);
+ }
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..cd5dce3
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,254 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..7d8c3e2
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,254 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..29dee36
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,259 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..2bb7875
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,259 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..539b395
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,254 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..edc063a
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,254 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..11f2641
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,259 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..ac3f801
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,259 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(vout);
+ vout = _mm_srli_si128(vout, 4);
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(vout);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..9a82755
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,259 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..3ae703b
--- /dev/null
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,259 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const int8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ const int8_t* a0 = a;
+ int8_t* c0 = c;
+ const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+ const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ a3 = a2;
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
+
+ w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+ w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+ }
+ }
+ }
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+
+ a0 = (const int8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const int8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const int8_t*) ((uintptr_t) a2 - kc);
+ a3 = (const int8_t*) ((uintptr_t) a3 - kc);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/MRx4c2-sse.c.in b/src/qs8-igemm/MRx4c2-sse.c.in
index d08425d..9603e3a 100644
--- a/src/qs8-igemm/MRx4c2-sse.c.in
+++ b/src/qs8-igemm/MRx4c2-sse.c.in
@@ -6,6 +6,7 @@
$assert SSE in [2, 3, 4]
$assert not XOP or AVX
$assert not AVX or SSE == 4
+$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
$assert VARIANT in ["LD64", "LD128"]
$assert MR <= 4
#include <assert.h>
@@ -25,9 +26,9 @@
#include <xnnpack/math.h>
-$PARAMS_STRUCT = "gemmlowp_sse4" if SSE >= 4 else "gemmlowp_sse2"
+$PARAMS_STRUCT = REQUANTIZATION.lower() + ("_sse4" if SSE >= 4 else "_sse2")
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
-void xnn_qs8_igemm_minmax_gemmlowp_ukernel_${MR}x4c2__${ISA}_${VARIANT.lower()}(
+void xnn_qs8_igemm_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x4c2__${ISA}_${VARIANT.lower()}(
size_t mr,
size_t nc,
size_t kc,
@@ -193,82 +194,93 @@
p -= ${MR} * sizeof(void*);
} while (p != 0);
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
+ $if REQUANTIZATION == "GEMMLOWP":
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
- $if SSE == 4:
+ $if SSE == 4:
+ $for M in range(MR):
+ const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
+ const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
+ $else:
+ $for M in range(MR):
+ const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
+
+ $for M in range(MR):
+ $if SSE >= 3:
+ const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
+ $else:
+ const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
+
+ $for M in range(MR):
+ const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
+ _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
$for M in range(MR):
- const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m128i vrem${M}x0123 =
+ _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
+
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $for M in range(MR):
+ vacc${M}x0123 =
+ _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ $elif REQUANTIZATION == "FP32":
+ $for M in range(MR):
+ __m128 vscaled${M}x0123 = _mm_cvtepi32_ps(vacc${M}x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_${"sse4" if SSE >= 4 else "sse2"}.scale);
+ $for M in range(MR):
+ vscaled${M}x0123 = _mm_mul_ps(vscaled${M}x0123, vscale);
$for M in range(MR):
- const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
- const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
- $else:
- $for M in range(MR):
- const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
-
- $for M in range(MR):
- $if SSE >= 3:
- const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
- $else:
- const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
-
- $for M in range(MR):
- const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
-
- $for M in range(MR):
- const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
- _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
-
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
- $for M in range(MR):
- const __m128i vrem${M}x0123 =
- _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
-
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
- $if M > 1:
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $else:
- const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $for M in range(MR):
- vacc${M}x0123 =
- _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ vacc${M}x0123 = _mm_cvtps_epi32(vscaled${M}x0123);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
$for M in range(0, MR, 2):
diff --git a/src/qs8-igemm/MRx4c8-sse.c.in b/src/qs8-igemm/MRx4c8-sse.c.in
index b616122..86ce128 100644
--- a/src/qs8-igemm/MRx4c8-sse.c.in
+++ b/src/qs8-igemm/MRx4c8-sse.c.in
@@ -6,6 +6,7 @@
$assert SSE in [2, 3, 4]
$assert not XOP or AVX
$assert not AVX or SSE == 4
+$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
$assert VARIANT in ["LD64", "LD128"]
$assert MR <= 4
#include <assert.h>
@@ -25,9 +26,9 @@
#include <xnnpack/math.h>
-$PARAMS_STRUCT = "gemmlowp_sse4" if SSE >= 4 else "gemmlowp_sse2"
+$PARAMS_STRUCT = REQUANTIZATION.lower() + ("_sse4" if SSE >= 4 else "_sse2")
$ISA = "xop" if XOP else "avx" if AVX else {2: "sse2", 3: "ssse3", 4: "sse41"}[SSE]
-void xnn_qs8_igemm_minmax_gemmlowp_ukernel_${MR}x4c8__${ISA}_${VARIANT.lower()}(
+void xnn_qs8_igemm_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x4c8__${ISA}_${VARIANT.lower()}(
size_t mr,
size_t nc,
size_t kc,
@@ -151,90 +152,101 @@
$for M in range(MR):
__m128i vacc${M}x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc${M}x02, vacc${M}x13), _mm_unpackhi_epi32(vacc${M}x02, vacc${M}x13));
- const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
- const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
+ $if REQUANTIZATION == "GEMMLOWP":
+ const __m128i vmultiplier = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.multiplier);
+ const __m128i vrounding = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.rounding);
- $if SSE == 4:
+ $if SSE == 4:
+ $for M in range(MR):
+ const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
+ const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
+ $else:
+ $for M in range(MR):
+ const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
+
+ $for M in range(MR):
+ $if SSE >= 3:
+ const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
+ $else:
+ const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
+
+ $for M in range(MR):
+ const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
+
+ $for M in range(MR):
+ const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+
+ $for M in range(MR):
+ const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
+ _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
+
+ $for M in range(MR):
+ const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
+
+ const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
$for M in range(MR):
- const __m128i vacc${M}x1133 = _mm_shuffle_epi32(vacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
+ const __m128i vrem${M}x0123 =
+ _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
+
+ const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
+ $if M > 1:
+ const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $else:
+ const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
+ $for M in range(MR):
+ vacc${M}x0123 =
+ _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ $elif REQUANTIZATION == "FP32":
+ $for M in range(MR):
+ __m128 vscaled${M}x0123 = _mm_cvtepi32_ps(vacc${M}x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_${"sse4" if SSE >= 4 else "sse2"}.scale);
+ $for M in range(MR):
+ vscaled${M}x0123 = _mm_mul_ps(vscaled${M}x0123, vscale);
$for M in range(MR):
- const __m128i vprod${M}x02 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x0123, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_add_epi64(_mm_mul_epi32(vacc${M}x1133, vmultiplier), vrounding);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(vprod${M}x02, 31);
- const __m128i vq31prod${M}x13 = _mm_add_epi64(vprod${M}x13, vprod${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_blend_epi16(vq31prod${M}x02, vq31prod${M}x13, 0xCC);
- $else:
- $for M in range(MR):
- const __m128i vnmask${M}x0123 = _mm_cmpgt_epi32(_mm_setzero_si128(), vacc${M}x0123);
-
- $for M in range(MR):
- $if SSE >= 3:
- const __m128i vabsacc${M}x0123 = _mm_abs_epi32(vacc${M}x0123);
- $else:
- const __m128i vabsacc${M}x0123 = _mm_sub_epi32(_mm_xor_si128(vacc${M}x0123, vnmask${M}x0123), vnmask${M}x0123);
-
- $for M in range(MR):
- const __m128i vabsacc${M}x1133 = _mm_shuffle_epi32(vabsacc${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vabsprod${M}x02 = _mm_mul_epu32(vabsacc${M}x0123, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x02 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(2, 2, 0, 0));
-
- $for M in range(MR):
- const __m128i vprod${M}x02 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x02, vnmask${M}x02), vnmask${M}x02);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x02 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x02, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vabsprod${M}x13 = _mm_mul_epu32(vabsacc${M}x1133, vmultiplier);
-
- $for M in range(MR):
- const __m128i vnmask${M}x13 = _mm_shuffle_epi32(vnmask${M}x0123, _MM_SHUFFLE(3, 3, 1, 1));
-
- $for M in range(MR):
- const __m128i vprod${M}x13 = _mm_sub_epi64(_mm_xor_si128(vabsprod${M}x13, vnmask${M}x13), vnmask${M}x13);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x13 = _mm_srli_epi64(_mm_add_epi64(vprod${M}x13, vrounding), 31);
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0213 = _mm_castps_si128(_mm_shuffle_ps(
- _mm_castsi128_ps(vq31prod${M}x02), _mm_castsi128_ps(vq31prod${M}x13), _MM_SHUFFLE(2, 0, 2, 0)));
-
- $for M in range(MR):
- const __m128i vq31prod${M}x0123 = _mm_shuffle_epi32(vq31prod${M}x0213, _MM_SHUFFLE(3, 1, 2, 0));
-
- const __m128i vremainder_mask = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_mask);
- $for M in range(MR):
- const __m128i vrem${M}x0123 =
- _mm_add_epi32(_mm_and_si128(vq31prod${M}x0123, vremainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), vq31prod${M}x0123));
-
- const __m128i vremainder_threshold = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.remainder_threshold);
- $if M > 1:
- const __m128i vshift = _mm_loadl_epi64((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $else:
- const __m128i vshift = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.shift);
- $for M in range(MR):
- vacc${M}x0123 =
- _mm_sub_epi32(_mm_sra_epi32(vq31prod${M}x0123, vshift), _mm_cmpgt_epi32(vrem${M}x0123, vremainder_threshold));
+ vacc${M}x0123 = _mm_cvtps_epi32(vscaled${M}x0123);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
$for M in range(0, MR, 2):
__m128i vacc${M}${min(M+1, MR-1)}x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123), voutput_zero_point);
$if SSE < 4:
- const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
- const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_max);
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x0123 = _mm_min_epi16(_mm_max_epi16(vacc${M}${min(M+1, MR-1)}x0123, voutput_min), voutput_max);
@@ -244,8 +256,8 @@
__m128i vout = _mm_packs_epi16(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123);
$if SSE == 4:
- vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
- vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_max));
if (nc >= 4) {
$for M in reversed(range(1, MR)):
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..1d8f14c
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,157 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..66ea6ea
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,157 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..ac4c742
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,158 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..6c892af
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,158 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..866740d
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,157 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..091c8f0
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,157 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..daf3c09
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,158 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..ffc8d56
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,158 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..eaa6e92
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,162 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..1cb8c87
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,162 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ }
+ }
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..a81f6a4
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c
@@ -0,0 +1,129 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..ead06e3
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c
@@ -0,0 +1,131 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..4360fb7
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,130 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..b1cffaf
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,132 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..ef57058
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,129 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..dfc49bf
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,131 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..cc9b9c2
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,130 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..c19d705
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,132 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc00x0123 = _mm_min_epi16(_mm_max_epi16(vacc00x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..406550e
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c
@@ -0,0 +1,134 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..2894376
--- /dev/null
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c
@@ -0,0 +1,136 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc00x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc0x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc00x0123, vacc00x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..ddc1725
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,194 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..a63c500
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,194 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..51cd3f6
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,195 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..6f16a59
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,195 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..1f99b51
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,194 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..3f96f5d
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,194 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..e4d15f9
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,195 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..f4d485b
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,195 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..00a4b2c
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,199 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..3710f0f
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,199 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ }
+ }
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..00d798f
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c
@@ -0,0 +1,159 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..56827ed
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c
@@ -0,0 +1,161 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..834cbdf
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,160 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..8f98520
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,162 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..e42f500
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,159 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..de772cb
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,161 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..d3f20da
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,160 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..88a422b
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,162 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..5c7af79
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c
@@ -0,0 +1,164 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..a725181
--- /dev/null
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c
@@ -0,0 +1,166 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc01x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..cde0021
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,232 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..cb62d7a
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,232 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..6385818
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,234 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..e5d00ed
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,234 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..02ef5f2
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,232 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..1ba142c
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,232 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..337b395
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,234 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..132a23e
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,234 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..0204f3a
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,237 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..2b600be
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,237 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ }
+ }
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..4116936
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c
@@ -0,0 +1,190 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..aa33476
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c
@@ -0,0 +1,192 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..bd5324f
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,192 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+ const __m128i vacc2x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x0, vacc2x2), _mm_unpackhi_epi32(vacc2x0, vacc2x2));
+ const __m128i vacc2x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x1, vacc2x3), _mm_unpackhi_epi32(vacc2x1, vacc2x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+ __m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..2a81f95
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,194 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x0, vacc0x2), _mm_unpackhi_epi32(vacc0x0, vacc0x2));
+ const __m128i vacc0x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x1, vacc0x3), _mm_unpackhi_epi32(vacc0x1, vacc0x3));
+ const __m128i vacc1x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x0, vacc1x2), _mm_unpackhi_epi32(vacc1x0, vacc1x2));
+ const __m128i vacc1x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x1, vacc1x3), _mm_unpackhi_epi32(vacc1x1, vacc1x3));
+ const __m128i vacc2x02 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x0, vacc2x2), _mm_unpackhi_epi32(vacc2x0, vacc2x2));
+ const __m128i vacc2x13 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x1, vacc2x3), _mm_unpackhi_epi32(vacc2x1, vacc2x3));
+
+ __m128i vacc0x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc0x02, vacc0x13), _mm_unpackhi_epi32(vacc0x02, vacc0x13));
+ __m128i vacc1x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc1x02, vacc1x13), _mm_unpackhi_epi32(vacc1x02, vacc1x13));
+ __m128i vacc2x0123 = _mm_add_epi32(_mm_unpacklo_epi32(vacc2x02, vacc2x13), _mm_unpackhi_epi32(vacc2x02, vacc2x13));
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..6531f50
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,190 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..500f75d
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,192 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..8831628
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,192 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..036b585
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,194 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
+ vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
+ vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
+ vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
+ vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
+ vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
+ vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
+ vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
+ vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->fp32_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc22x0123 = _mm_min_epi16(_mm_max_epi16(vacc22x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..883264f
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c
@@ -0,0 +1,195 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
+ vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+ vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
+ vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..439051b
--- /dev/null
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c
@@ -0,0 +1,197 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c8-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ __m128i vacc0x0 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[0]);
+ __m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
+ __m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
+ __m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
+ __m128i vacc1x0 = vacc0x0;
+ __m128i vacc1x1 = vacc0x1;
+ __m128i vacc1x2 = vacc0x2;
+ __m128i vacc1x3 = vacc0x3;
+ __m128i vacc2x0 = vacc0x0;
+ __m128i vacc2x1 = vacc0x1;
+ __m128i vacc2x2 = vacc0x2;
+ __m128i vacc2x3 = vacc0x3;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
+ vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
+ vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
+ vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
+ vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
+ vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
+ vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
+ vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
+ vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k += 8 * sizeof(int8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m128i vacc0x01 = _mm_hadd_epi32(vacc0x0, vacc0x1);
+ const __m128i vacc0x23 = _mm_hadd_epi32(vacc0x2, vacc0x3);
+ const __m128i vacc1x01 = _mm_hadd_epi32(vacc1x0, vacc1x1);
+ const __m128i vacc1x23 = _mm_hadd_epi32(vacc1x2, vacc1x3);
+ const __m128i vacc2x01 = _mm_hadd_epi32(vacc2x0, vacc2x1);
+ const __m128i vacc2x23 = _mm_hadd_epi32(vacc2x2, vacc2x3);
+
+ __m128i vacc0x0123 = _mm_hadd_epi32(vacc0x01, vacc0x23);
+ __m128i vacc1x0123 = _mm_hadd_epi32(vacc1x01, vacc1x23);
+ __m128i vacc2x0123 = _mm_hadd_epi32(vacc2x01, vacc2x23);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc22x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc2x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc22x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->fp32_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c
new file mode 100644
index 0000000..10c39c7
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c
@@ -0,0 +1,269 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__avx_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c
new file mode 100644
index 0000000..c1fd952
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c
@@ -0,0 +1,269 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__avx_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
new file mode 100644
index 0000000..31299ee
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
@@ -0,0 +1,271 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse2_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
new file mode 100644
index 0000000..ea52094
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
@@ -0,0 +1,271 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <emmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse2_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
new file mode 100644
index 0000000..1ba4d7d
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
@@ -0,0 +1,269 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse41_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
new file mode 100644
index 0000000..f383fa7
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
@@ -0,0 +1,269 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <smmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse41_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c
new file mode 100644
index 0000000..9587ef0
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld128.c
@@ -0,0 +1,271 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c
new file mode 100644
index 0000000..363b742
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-ssse3-ld64.c
@@ -0,0 +1,271 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <tmmintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_add_epi32(vacc0x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc1x0123 = _mm_add_epi32(vacc1x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc2x0123 = _mm_add_epi32(vacc2x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ vacc3x0123 = _mm_add_epi32(vacc3x0123,
+ _mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse2.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+ const __m128i voutput_min = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_min);
+ const __m128i voutput_max = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_max);
+ vacc01x0123 = _mm_min_epi16(_mm_max_epi16(vacc01x0123, voutput_min), voutput_max);
+ vacc23x0123 = _mm_min_epi16(_mm_max_epi16(vacc23x0123, voutput_min), voutput_max);
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(3, 3, 3, 3)));
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(2, 2, 2, 2)));
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_cvtsi128_si32(_mm_shuffle_epi32(vout, _MM_SHUFFLE(1, 1, 1, 1)));
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi16(vout, 6);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi16(vout, 4);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi16(vout, 2);
+ *((int8_t*) c0) = (int8_t) _mm_cvtsi128_si32(vout);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c
new file mode 100644
index 0000000..08bc5cd
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c
@@ -0,0 +1,274 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__xop_ld128(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
+ const __m128i vsb01 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb01);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb01, vsb01);
+ const __m128i vxb1 = _mm_unpackhi_epi8(vb01, vsb01);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
+ const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c
new file mode 100644
index 0000000..f660147
--- /dev/null
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c
@@ -0,0 +1,274 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx4c2-sse.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+ #include <x86intrin.h>
+#else
+ #include <immintrin.h>
+ #include <ammintrin.h>
+#endif
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__xop_ld64(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const int8_t** restrict a,
+ const void* restrict w,
+ int8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 4);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (4 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(int8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 2);
+ int8_t* c0 = c;
+ int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+ int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 4) {
+ c3 = c2;
+ }
+
+ do {
+ __m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
+ __m128i vacc1x0123 = vacc0x0123;
+ __m128i vacc2x0123 = vacc0x0123;
+ __m128i vacc3x0123 = vacc0x0123;
+ w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+
+ size_t p = ks;
+ do {
+ const int8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const int8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const int8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ const int8_t* restrict a3 = a[3];
+ if XNN_UNPREDICTABLE(a3 != zero) {
+ a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
+ }
+ a += 4;
+
+ size_t k = kc;
+ while (k >= 8 * sizeof(int8_t)) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 += 8;
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 += 8;
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
+
+ w = (const void*) ((uintptr_t) w + 32);
+ k -= 8 * sizeof(int8_t);
+ }
+ if (k != 0) {
+ const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
+ const __m128i vxa0 = _mm_cvtepi8_epi16(va0);
+ a0 = (const int8_t*) ((uintptr_t) a0 + k);
+ const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
+ const __m128i vxa1 = _mm_cvtepi8_epi16(va1);
+ a1 = (const int8_t*) ((uintptr_t) a1 + k);
+ const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
+ const __m128i vxa2 = _mm_cvtepi8_epi16(va2);
+ a2 = (const int8_t*) ((uintptr_t) a2 + k);
+ const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
+ const __m128i vxa3 = _mm_cvtepi8_epi16(va3);
+ a3 = (const int8_t*) ((uintptr_t) a3 + k);
+
+ const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
+
+ if (k > 2 * sizeof(int8_t)) {
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
+
+ if (k > 4 * sizeof(int8_t)) {
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
+ w = (const void*) ((uintptr_t) w + 8);
+ const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+
+ vacc0x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
+ vacc1x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
+ vacc2x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
+ vacc3x0123 = _mm_maddd_epi16(
+ _mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
+ }
+ }
+ }
+ p -= 4 * sizeof(void*);
+ } while (p != 0);
+
+ __m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
+ __m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
+ __m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
+ __m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
+
+ const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
+ vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale);
+ vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale);
+ vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale);
+ vscaled3x0123 = _mm_mul_ps(vscaled3x0123, vscale);
+
+ vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
+ vacc1x0123 = _mm_cvtps_epi32(vscaled1x0123);
+ vacc2x0123 = _mm_cvtps_epi32(vscaled2x0123);
+ vacc3x0123 = _mm_cvtps_epi32(vscaled3x0123);
+
+ const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
+ __m128i vacc01x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0x0123, vacc1x0123), voutput_zero_point);
+ __m128i vacc23x0123 = _mm_adds_epi16(_mm_packs_epi32(vacc2x0123, vacc3x0123), voutput_zero_point);
+
+
+ __m128i vout = _mm_packs_epi16(vacc01x0123, vacc23x0123);
+
+ vout = _mm_max_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_min));
+ vout = _mm_min_epi8(vout, _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_max));
+
+ if (nc >= 4) {
+ *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout, 3);
+ c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout, 2);
+ c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
+ *((uint32_t*) c1) = (uint32_t) _mm_extract_epi32(vout, 1);
+ c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
+ *((uint32_t*) c0) = (uint32_t) _mm_cvtsi128_si32(vout);
+ c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const int8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 4;
+ } else {
+ if (nc & 2) {
+ *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout, 6);
+ c3 += 2;
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout, 4);
+ c2 += 2;
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout, 2);
+ c1 += 2;
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout, 0);
+ c0 += 2;
+ vout = _mm_srli_epi32(vout, 16);
+ }
+ if (nc & 1) {
+ *((int8_t*) c3) = (int8_t) _mm_extract_epi8(vout, 12);
+ *((int8_t*) c2) = (int8_t) _mm_extract_epi8(vout, 8);
+ *((int8_t*) c1) = (int8_t) _mm_extract_epi8(vout, 4);
+ *((int8_t*) c0) = (int8_t) _mm_extract_epi8(vout, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/xnnpack/gemm.h b/src/xnnpack/gemm.h
index 445c098..b57e3ca 100644
--- a/src/xnnpack/gemm.h
+++ b/src/xnnpack/gemm.h
@@ -637,91 +637,181 @@
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__sse2_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__sse2_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__ssse3_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__ssse3_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__ssse3_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__ssse3_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__sse41_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__sse41_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__sse41_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__sse41_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__avx_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__avx_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__avx_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__avx_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__xop_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__xop_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__xop_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__xop_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__sse2_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__sse2_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__sse2_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__sse2_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__ssse3_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__ssse3_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__ssse3_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__ssse3_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__sse41_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__sse41_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__sse41_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__sse41_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__sse41_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__sse41_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__sse41_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__sse41_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__avx_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__avx_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__avx_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__avx_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__avx_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__avx_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__avx_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__avx_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c2__xop_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c2__xop_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c2__xop_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x4c2__xop_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c2__xop_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c2__xop_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c2__xop_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x4c2__xop_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__sse2_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__sse2_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse2_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__ssse3_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__ssse3_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__ssse3_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__sse41_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__sse41_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse41_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__avx_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__avx_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__avx_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__xop_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__xop_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__xop_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld64)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__sse2_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__sse2_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse2_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse2_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__ssse3_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__ssse3_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__ssse3_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__sse41_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__sse41_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__sse41_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__sse41_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__avx_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__avx_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__avx_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__avx_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x4c8__xop_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x4c8__xop_ld128)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x4c8__xop_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x8c8__avx2)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_2x8c8__avx2)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_3x8c8__avx2)
diff --git a/src/xnnpack/igemm.h b/src/xnnpack/igemm.h
index ade91b9..eba12e4 100644
--- a/src/xnnpack/igemm.h
+++ b/src/xnnpack/igemm.h
@@ -440,91 +440,181 @@
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__sse2_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__sse2_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse2_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse2_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse2_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__ssse3_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__ssse3_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__ssse3_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__ssse3_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__ssse3_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__ssse3_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__ssse3_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__sse41_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__sse41_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__sse41_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__sse41_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse41_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse41_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse41_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__avx_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__avx_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__avx_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__avx_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__avx_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__avx_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__avx_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__xop_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__xop_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__xop_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__xop_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__xop_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__xop_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__xop_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__sse2_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__sse2_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__sse2_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__sse2_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse2_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse2_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse2_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__ssse3_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__ssse3_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__ssse3_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__ssse3_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__ssse3_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__ssse3_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__ssse3_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__ssse3_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__sse41_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__sse41_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__sse41_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__sse41_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__sse41_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__sse41_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__sse41_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__sse41_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__avx_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__avx_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__avx_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__avx_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__avx_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__avx_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__avx_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__avx_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c2__xop_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c2__xop_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c2__xop_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x4c2__xop_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c2__xop_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c2__xop_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c2__xop_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x4c2__xop_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__sse2_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__sse2_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__sse2_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse2_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__ssse3_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__ssse3_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__ssse3_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__ssse3_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__ssse3_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__sse41_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__sse41_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__sse41_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse41_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__avx_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__avx_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__avx_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__avx_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__xop_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__xop_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__xop_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld64)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__xop_ld64)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__sse2_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__sse2_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__sse2_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse2_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__ssse3_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__ssse3_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__ssse3_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__ssse3_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__ssse3_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__ssse3_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__sse41_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__sse41_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__sse41_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__sse41_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__avx_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__avx_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__avx_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__avx_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x4c8__xop_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x4c8__xop_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x4c8__xop_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld128)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_3x4c8__xop_ld128)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x8c8__avx2)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_2x8c8__avx2)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_3x8c8__avx2)
diff --git a/src/xnnpack/params-init.h b/src/xnnpack/params-init.h
index 2d81c53..885fa69 100644
--- a/src/xnnpack/params-init.h
+++ b/src/xnnpack/params-init.h
@@ -354,6 +354,42 @@
}
}
+static inline void xnn_init_qs8_conv_minmax_fp32_sse2_params(
+ union xnn_qs8_conv_minmax_params params[XNN_MIN_ELEMENTS(1)],
+ float scale,
+ int8_t output_zero_point,
+ int8_t output_min,
+ int8_t output_max)
+{
+ for (uint32_t i = 0; i < 4; i++) {
+ params->fp32_sse2.scale[i] = scale;
+ }
+ for (uint32_t i = 0; i < 8; i++) {
+ params->fp32_sse2.output_zero_point[i] = (int16_t) output_zero_point;
+ params->fp32_sse2.output_min[i] = (int16_t) output_min;
+ params->fp32_sse2.output_max[i] = (int16_t) output_max;
+ }
+}
+
+static inline void xnn_init_qs8_conv_minmax_fp32_sse4_params(
+ union xnn_qs8_conv_minmax_params params[XNN_MIN_ELEMENTS(1)],
+ float scale,
+ int8_t output_zero_point,
+ int8_t output_min,
+ int8_t output_max)
+{
+ for (uint32_t i = 0; i < 4; i++) {
+ params->fp32_sse4.scale[i] = scale;
+ }
+ for (uint32_t i = 0; i < 8; i++) {
+ params->fp32_sse4.output_zero_point[i] = (int16_t) output_zero_point;
+ }
+ for (uint32_t i = 0; i < 16; i++) {
+ params->fp32_sse4.output_min[i] = output_min;
+ params->fp32_sse4.output_max[i] = output_max;
+ }
+}
+
static inline void xnn_init_qs8_conv_minmax_fp32_avx2_params(
union xnn_qs8_conv_minmax_params params[XNN_MIN_ELEMENTS(1)],
float scale,