QU8 GEMM/IGEMM AVX2 microkernels with FP32 requantization
PiperOrigin-RevId: 382830855
diff --git a/BUILD.bazel b/BUILD.bazel
index 0486f80..8d4eec5 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -3971,6 +3971,12 @@
"src/qs8-vaddc/gen/minmax-avx2-mul32-ld64-x16.c",
"src/qs8-vaddc/gen/minmax-avx2-mul32-ld64-x24.c",
"src/qs8-vaddc/gen/minmax-avx2-mul32-ld64-x32.c",
+ "src/qu8-gemm/gen/1x8c8-minmax-fp32-avx2.c",
+ "src/qu8-gemm/gen/2x8c8-minmax-fp32-avx2.c",
+ "src/qu8-gemm/gen/3x8c8-minmax-fp32-avx2.c",
+ "src/qu8-igemm/gen/1x8c8-minmax-fp32-avx2.c",
+ "src/qu8-igemm/gen/2x8c8-minmax-fp32-avx2.c",
+ "src/qu8-igemm/gen/3x8c8-minmax-fp32-avx2.c",
]
AVX512F_UKERNELS = [
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ca0b93c..7dcfde3 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -3166,7 +3166,13 @@
src/qs8-vaddc/gen/minmax-avx2-mul32-ld64-x8.c
src/qs8-vaddc/gen/minmax-avx2-mul32-ld64-x16.c
src/qs8-vaddc/gen/minmax-avx2-mul32-ld64-x24.c
- src/qs8-vaddc/gen/minmax-avx2-mul32-ld64-x32.c)
+ src/qs8-vaddc/gen/minmax-avx2-mul32-ld64-x32.c
+ src/qu8-gemm/gen/1x8c8-minmax-fp32-avx2.c
+ src/qu8-gemm/gen/2x8c8-minmax-fp32-avx2.c
+ src/qu8-gemm/gen/3x8c8-minmax-fp32-avx2.c
+ src/qu8-igemm/gen/1x8c8-minmax-fp32-avx2.c
+ src/qu8-igemm/gen/2x8c8-minmax-fp32-avx2.c
+ src/qu8-igemm/gen/3x8c8-minmax-fp32-avx2.c)
SET(XNNPACK_AVX512F_MICROKERNEL_SRCS
src/f32-dwconv/gen/up16x4-minmax-avx512f-acc2.c
diff --git a/scripts/generate-qs8-gemm.sh b/scripts/generate-qs8-gemm.sh
index d8043c0..ac2a709 100755
--- a/scripts/generate-qs8-gemm.sh
+++ b/scripts/generate-qs8-gemm.sh
@@ -594,26 +594,30 @@
tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=2 -D VARIANT=LD128 -D DATATYPE=QS8 -D REQUANTIZATION=GEMMLOWP -o src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-avx2.c
tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=3 -D VARIANT=LD128 -D DATATYPE=QS8 -D REQUANTIZATION=GEMMLOWP -o src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-avx2.c
+tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=1 -D VARIANT=LD128 -D DATATYPE=QC8 -D REQUANTIZATION=FP32 -o src/qc8-gemm/gen/1x8c8-minmax-fp32-avx2.c
+tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=2 -D VARIANT=LD128 -D DATATYPE=QC8 -D REQUANTIZATION=FP32 -o src/qc8-gemm/gen/2x8c8-minmax-fp32-avx2.c
+tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=3 -D VARIANT=LD128 -D DATATYPE=QC8 -D REQUANTIZATION=FP32 -o src/qc8-gemm/gen/3x8c8-minmax-fp32-avx2.c
+
tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=1 -D VARIANT=LD128 -D DATATYPE=QS8 -D REQUANTIZATION=FP32 -o src/qs8-gemm/gen/1x8c8-minmax-fp32-avx2.c
tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=2 -D VARIANT=LD128 -D DATATYPE=QS8 -D REQUANTIZATION=FP32 -o src/qs8-gemm/gen/2x8c8-minmax-fp32-avx2.c
tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=3 -D VARIANT=LD128 -D DATATYPE=QS8 -D REQUANTIZATION=FP32 -o src/qs8-gemm/gen/3x8c8-minmax-fp32-avx2.c
-tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=1 -D VARIANT=LD128 -D DATATYPE=QC8 -D REQUANTIZATION=FP32 -o src/qc8-gemm/gen/1x8c8-minmax-fp32-avx2.c
-tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=2 -D VARIANT=LD128 -D DATATYPE=QC8 -D REQUANTIZATION=FP32 -o src/qc8-gemm/gen/2x8c8-minmax-fp32-avx2.c
-tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=3 -D VARIANT=LD128 -D DATATYPE=QC8 -D REQUANTIZATION=FP32 -o src/qc8-gemm/gen/3x8c8-minmax-fp32-avx2.c
+tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=1 -D VARIANT=LD128 -D DATATYPE=QU8 -D REQUANTIZATION=FP32 -o src/qu8-gemm/gen/1x8c8-minmax-fp32-avx2.c
+tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=2 -D VARIANT=LD128 -D DATATYPE=QU8 -D REQUANTIZATION=FP32 -o src/qu8-gemm/gen/2x8c8-minmax-fp32-avx2.c
+tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=3 -D VARIANT=LD128 -D DATATYPE=QU8 -D REQUANTIZATION=FP32 -o src/qu8-gemm/gen/3x8c8-minmax-fp32-avx2.c
tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=1 -D VARIANT=EXTENDED -D DATATYPE=QS8 -D REQUANTIZATION=GEMMLOWP -o src/qs8-gemm/gen/1x8c8-xw-minmax-gemmlowp-avx2.c
tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=2 -D VARIANT=EXTENDED -D DATATYPE=QS8 -D REQUANTIZATION=GEMMLOWP -o src/qs8-gemm/gen/2x8c8-xw-minmax-gemmlowp-avx2.c
tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=3 -D VARIANT=EXTENDED -D DATATYPE=QS8 -D REQUANTIZATION=GEMMLOWP -o src/qs8-gemm/gen/3x8c8-xw-minmax-gemmlowp-avx2.c
-tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=1 -D VARIANT=EXTENDED -D DATATYPE=QS8 -D REQUANTIZATION=FP32 -o src/qs8-gemm/gen/1x8c8-xw-minmax-fp32-avx2.c
-tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=2 -D VARIANT=EXTENDED -D DATATYPE=QS8 -D REQUANTIZATION=FP32 -o src/qs8-gemm/gen/2x8c8-xw-minmax-fp32-avx2.c
-tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=3 -D VARIANT=EXTENDED -D DATATYPE=QS8 -D REQUANTIZATION=FP32 -o src/qs8-gemm/gen/3x8c8-xw-minmax-fp32-avx2.c
-
tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=1 -D VARIANT=EXTENDED -D DATATYPE=QC8 -D REQUANTIZATION=FP32 -o src/qc8-gemm/gen/1x8c8-xw-minmax-fp32-avx2.c
tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=2 -D VARIANT=EXTENDED -D DATATYPE=QC8 -D REQUANTIZATION=FP32 -o src/qc8-gemm/gen/2x8c8-xw-minmax-fp32-avx2.c
tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=3 -D VARIANT=EXTENDED -D DATATYPE=QC8 -D REQUANTIZATION=FP32 -o src/qc8-gemm/gen/3x8c8-xw-minmax-fp32-avx2.c
+tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=1 -D VARIANT=EXTENDED -D DATATYPE=QS8 -D REQUANTIZATION=FP32 -o src/qs8-gemm/gen/1x8c8-xw-minmax-fp32-avx2.c
+tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=2 -D VARIANT=EXTENDED -D DATATYPE=QS8 -D REQUANTIZATION=FP32 -o src/qs8-gemm/gen/2x8c8-xw-minmax-fp32-avx2.c
+tools/xngen src/qs8-gemm/MRx8c8-avx2.c.in -D MR=3 -D VARIANT=EXTENDED -D DATATYPE=QS8 -D REQUANTIZATION=FP32 -o src/qs8-gemm/gen/3x8c8-xw-minmax-fp32-avx2.c
+
################################## x86 AVX512 #################################
### C8 micro-kernels
tools/xngen src/qs8-gemm/MRx16c8-avx512skx.c.in -D MR=1 -D VARIANT=LD256 -D DATATYPE=QS8 -D REQUANTIZATION=GEMMLOWP -o src/qs8-gemm/gen/1x16c8-minmax-gemmlowp-avx512skx.c
diff --git a/scripts/generate-qs8-igemm.sh b/scripts/generate-qs8-igemm.sh
index 65853f9..4cba94e 100755
--- a/scripts/generate-qs8-igemm.sh
+++ b/scripts/generate-qs8-igemm.sh
@@ -534,13 +534,17 @@
tools/xngen src/qs8-igemm/MRx8c8-avx2.c.in -D MR=2 -D DATATYPE=QS8 -D REQUANTIZATION=GEMMLOWP -o src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-avx2.c
tools/xngen src/qs8-igemm/MRx8c8-avx2.c.in -D MR=3 -D DATATYPE=QS8 -D REQUANTIZATION=GEMMLOWP -o src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-avx2.c
+tools/xngen src/qs8-igemm/MRx8c8-avx2.c.in -D MR=1 -D DATATYPE=QC8 -D REQUANTIZATION=FP32 -o src/qc8-igemm/gen/1x8c8-minmax-fp32-avx2.c
+tools/xngen src/qs8-igemm/MRx8c8-avx2.c.in -D MR=2 -D DATATYPE=QC8 -D REQUANTIZATION=FP32 -o src/qc8-igemm/gen/2x8c8-minmax-fp32-avx2.c
+tools/xngen src/qs8-igemm/MRx8c8-avx2.c.in -D MR=3 -D DATATYPE=QC8 -D REQUANTIZATION=FP32 -o src/qc8-igemm/gen/3x8c8-minmax-fp32-avx2.c
+
tools/xngen src/qs8-igemm/MRx8c8-avx2.c.in -D MR=1 -D DATATYPE=QS8 -D REQUANTIZATION=FP32 -o src/qs8-igemm/gen/1x8c8-minmax-fp32-avx2.c
tools/xngen src/qs8-igemm/MRx8c8-avx2.c.in -D MR=2 -D DATATYPE=QS8 -D REQUANTIZATION=FP32 -o src/qs8-igemm/gen/2x8c8-minmax-fp32-avx2.c
tools/xngen src/qs8-igemm/MRx8c8-avx2.c.in -D MR=3 -D DATATYPE=QS8 -D REQUANTIZATION=FP32 -o src/qs8-igemm/gen/3x8c8-minmax-fp32-avx2.c
-tools/xngen src/qs8-igemm/MRx8c8-avx2.c.in -D MR=1 -D DATATYPE=QC8 -D REQUANTIZATION=FP32 -o src/qc8-igemm/gen/1x8c8-minmax-fp32-avx2.c
-tools/xngen src/qs8-igemm/MRx8c8-avx2.c.in -D MR=2 -D DATATYPE=QC8 -D REQUANTIZATION=FP32 -o src/qc8-igemm/gen/2x8c8-minmax-fp32-avx2.c
-tools/xngen src/qs8-igemm/MRx8c8-avx2.c.in -D MR=3 -D DATATYPE=QC8 -D REQUANTIZATION=FP32 -o src/qc8-igemm/gen/3x8c8-minmax-fp32-avx2.c
+tools/xngen src/qs8-igemm/MRx8c8-avx2.c.in -D MR=1 -D DATATYPE=QU8 -D REQUANTIZATION=FP32 -o src/qu8-igemm/gen/1x8c8-minmax-fp32-avx2.c
+tools/xngen src/qs8-igemm/MRx8c8-avx2.c.in -D MR=2 -D DATATYPE=QU8 -D REQUANTIZATION=FP32 -o src/qu8-igemm/gen/2x8c8-minmax-fp32-avx2.c
+tools/xngen src/qs8-igemm/MRx8c8-avx2.c.in -D MR=3 -D DATATYPE=QU8 -D REQUANTIZATION=FP32 -o src/qu8-igemm/gen/3x8c8-minmax-fp32-avx2.c
################################## x86 AVX512 #################################
### C8 micro-kernels
diff --git a/src/init.c b/src/init.c
index 2f0cd3d..3519c1f 100644
--- a/src/init.c
+++ b/src/init.c
@@ -2107,12 +2107,53 @@
#ifndef XNN_NO_QU8_OPERATORS
init_flags |= XNN_INIT_FLAG_QU8;
- xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_4x4c2__sse2_ld64);
- xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_4x4c2__sse2_ld64);
- xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
- xnn_params.qu8.gemm.mr = 4;
- xnn_params.qu8.gemm.nr = 4;
- xnn_params.qu8.gemm.log2_kr = 1;
+ if (cpuinfo_has_x86_xop()) {
+ // XOP should be checked before AVX2: AMD Excavator supports both, but performs better with XOP microkernels
+ xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld64);
+ xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld64);
+ xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x4c8__xop_ld64);
+ xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x4c8__xop_ld64);
+ xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
+ xnn_params.qu8.gemm.mr = 2;
+ xnn_params.qu8.gemm.nr = 4;
+ xnn_params.qu8.gemm.log2_kr = 3;
+ } else if (cpuinfo_has_x86_avx2()) {
+ xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2);
+ xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2);
+ xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2);
+ xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2);
+ xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_avx2_params;
+ xnn_params.qu8.gemm.mr = 3;
+ xnn_params.qu8.gemm.nr = 8;
+ xnn_params.qu8.gemm.log2_kr = 3;
+ } else if (cpuinfo_has_x86_avx()) {
+ xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_2x4c8__avx_ld128);
+ xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_2x4c8__avx_ld128);
+ xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x4c8__avx_ld128);
+ xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x4c8__avx_ld128);
+ xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
+ xnn_params.qu8.gemm.mr = 2;
+ xnn_params.qu8.gemm.nr = 4;
+ xnn_params.qu8.gemm.log2_kr = 3;
+ } else if (cpuinfo_has_x86_sse4_1()) {
+ xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_3x4c8__sse41_ld64);
+ xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_3x4c8__sse41_ld64);
+ xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x4c8__sse41_ld64);
+ xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x4c8__sse41_ld64);
+ xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
+ xnn_params.qu8.gemm.mr = 3;
+ xnn_params.qu8.gemm.nr = 4;
+ xnn_params.qu8.gemm.log2_kr = 3;
+ } else {
+ xnn_params.qu8.gemm.minmax.gemm = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_3x4c8__sse2_ld64);
+ xnn_params.qu8.gemm.minmax.igemm = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_3x4c8__sse2_ld64);
+ xnn_params.qu8.gemm.minmax.gemm1 = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_function) xnn_qu8_gemm_minmax_fp32_ukernel_1x4c8__sse2_ld64);
+ xnn_params.qu8.gemm.minmax.igemm1 = xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_function) xnn_qu8_igemm_minmax_fp32_ukernel_1x4c8__sse2_ld64);
+ xnn_params.qu8.gemm.init.qu8 = xnn_init_qu8_conv_minmax_fp32_sse2_params;
+ xnn_params.qu8.gemm.mr = 3;
+ xnn_params.qu8.gemm.nr = 4;
+ xnn_params.qu8.gemm.log2_kr = 3;
+ }
xnn_params.qu8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qu8_dwconv_minmax_gemmlowp_ukernel_up8x9__sse2;
xnn_params.qu8.dwconv[0].init.qu8 = xnn_init_qu8_conv_minmax_gemmlowp_sse2_params;
diff --git a/src/params-init.c b/src/params-init.c
index d7db75c..9a8bc9b 100644
--- a/src/params-init.c
+++ b/src/params-init.c
@@ -122,6 +122,27 @@
params->fp32_sse2.output_max[i] = output_max;
}
}
+
+void xnn_init_qu8_conv_minmax_fp32_avx2_params(
+ union xnn_qu8_conv_minmax_params params[XNN_MIN_ELEMENTS(1)],
+ uint8_t kernel_zero_point,
+ float scale,
+ uint8_t output_zero_point,
+ uint8_t output_min,
+ uint8_t output_max)
+{
+ for (uint32_t i = 0; i < 8; i++) {
+ params->fp32_avx2.scale[i] = scale;
+ }
+ for (uint32_t i = 0; i < 16; i++) {
+ params->fp32_avx2.kernel_zero_point[i] = (int16_t) (uint16_t) kernel_zero_point;
+ params->fp32_avx2.output_zero_point[i] = (int16_t) (uint16_t) output_zero_point;
+ }
+ for (uint32_t i = 0; i < 32; i++) {
+ params->fp32_avx2.output_min[i] = output_min;
+ params->fp32_avx2.output_max[i] = output_max;
+ }
+}
#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
#if XNN_ARCH_ARM || XNN_ARCH_ARM64
diff --git a/src/qc8-gemm/gen/1x8c8-minmax-fp32-avx2.c b/src/qc8-gemm/gen/1x8c8-minmax-fp32-avx2.c
index 6c7ede0..6390bb4 100644
--- a/src/qc8-gemm/gen/1x8c8-minmax-fp32-avx2.c
+++ b/src/qc8-gemm/gen/1x8c8-minmax-fp32-avx2.c
@@ -43,18 +43,18 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -66,20 +66,20 @@
const __m256i vxb01 = _mm256_cvtepi8_epi16(vb01);
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
@@ -94,7 +94,7 @@
__m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
const __m256 vscale01234567 = _mm256_load_ps(w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ w = (const void*) ((const float*) w + 8);
vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale01234567);
vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
diff --git a/src/qc8-gemm/gen/1x8c8-xw-minmax-fp32-avx2.c b/src/qc8-gemm/gen/1x8c8-xw-minmax-fp32-avx2.c
index 2f43afd..d6ce306 100644
--- a/src/qc8-gemm/gen/1x8c8-xw-minmax-fp32-avx2.c
+++ b/src/qc8-gemm/gen/1x8c8-xw-minmax-fp32-avx2.c
@@ -43,18 +43,18 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -65,17 +65,17 @@
const __m256i vxb01 = _mm256_load_si256((const __m256i*) w);
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
- const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 16));
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
- const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 32 * sizeof(int16_t)));
+ const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 32));
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
- const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 48 * sizeof(int16_t)));
+ const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 48));
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 64);
k += 8 * sizeof(int8_t);
}
@@ -90,7 +90,7 @@
__m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
const __m256 vscale01234567 = _mm256_load_ps(w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ w = (const void*) ((const float*) w + 8);
vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale01234567);
vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
diff --git a/src/qc8-gemm/gen/2x8c8-minmax-fp32-avx2.c b/src/qc8-gemm/gen/2x8c8-minmax-fp32-avx2.c
index 6cf5e02..7c43f54 100644
--- a/src/qc8-gemm/gen/2x8c8-minmax-fp32-avx2.c
+++ b/src/qc8-gemm/gen/2x8c8-minmax-fp32-avx2.c
@@ -49,22 +49,22 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
__m256i vacc1x45 = vacc0x45;
__m256i vacc1x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -80,23 +80,23 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
@@ -116,7 +116,7 @@
__m256 vscaled1x01234567 = _mm256_cvtepi32_ps(vacc1x01234567);
const __m256 vscale01234567 = _mm256_load_ps(w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ w = (const void*) ((const float*) w + 8);
vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale01234567);
vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale01234567);
@@ -170,7 +170,7 @@
}
if (nc & 1) {
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
}
nc = 0;
diff --git a/src/qc8-gemm/gen/2x8c8-xw-minmax-fp32-avx2.c b/src/qc8-gemm/gen/2x8c8-xw-minmax-fp32-avx2.c
index a15d199..051d39c 100644
--- a/src/qc8-gemm/gen/2x8c8-xw-minmax-fp32-avx2.c
+++ b/src/qc8-gemm/gen/2x8c8-xw-minmax-fp32-avx2.c
@@ -49,22 +49,22 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
__m256i vacc1x45 = vacc0x45;
__m256i vacc1x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -79,20 +79,20 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
- const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 16));
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
- const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 32 * sizeof(int16_t)));
+ const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 32));
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
- const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 48 * sizeof(int16_t)));
+ const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 48));
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 64);
k += 8 * sizeof(int8_t);
}
@@ -112,7 +112,7 @@
__m256 vscaled1x01234567 = _mm256_cvtepi32_ps(vacc1x01234567);
const __m256 vscale01234567 = _mm256_load_ps(w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ w = (const void*) ((const float*) w + 8);
vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale01234567);
vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale01234567);
@@ -166,7 +166,7 @@
}
if (nc & 1) {
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
}
nc = 0;
diff --git a/src/qc8-gemm/gen/3x8c8-minmax-fp32-avx2.c b/src/qc8-gemm/gen/3x8c8-minmax-fp32-avx2.c
index f7a699e..13f50b1 100644
--- a/src/qc8-gemm/gen/3x8c8-minmax-fp32-avx2.c
+++ b/src/qc8-gemm/gen/3x8c8-minmax-fp32-avx2.c
@@ -55,16 +55,16 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
@@ -74,7 +74,7 @@
__m256i vacc2x23 = vacc0x23;
__m256i vacc2x45 = vacc0x45;
__m256i vacc2x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -94,26 +94,26 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
vacc2x01 = _mm256_add_epi32(vacc2x01, _mm256_madd_epi16(vxa2, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
vacc2x23 = _mm256_add_epi32(vacc2x23, _mm256_madd_epi16(vxa2, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
vacc2x45 = _mm256_add_epi32(vacc2x45, _mm256_madd_epi16(vxa2, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
vacc2x67 = _mm256_add_epi32(vacc2x67, _mm256_madd_epi16(vxa2, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
@@ -138,7 +138,7 @@
__m256 vscaled2x01234567 = _mm256_cvtepi32_ps(vacc2x01234567);
const __m256 vscale01234567 = _mm256_load_ps(w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ w = (const void*) ((const float*) w + 8);
vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale01234567);
vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale01234567);
vscaled2x01234567 = _mm256_mul_ps(vscaled2x01234567, vscale01234567);
@@ -203,8 +203,8 @@
}
if (nc & 1) {
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
- *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
+ *c2 = (int8_t) _mm_extract_epi8(vout_lo, 8);
}
nc = 0;
diff --git a/src/qc8-gemm/gen/3x8c8-xw-minmax-fp32-avx2.c b/src/qc8-gemm/gen/3x8c8-xw-minmax-fp32-avx2.c
index 1e70802..bf89e3c 100644
--- a/src/qc8-gemm/gen/3x8c8-xw-minmax-fp32-avx2.c
+++ b/src/qc8-gemm/gen/3x8c8-xw-minmax-fp32-avx2.c
@@ -55,16 +55,16 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
@@ -74,7 +74,7 @@
__m256i vacc2x23 = vacc0x23;
__m256i vacc2x45 = vacc0x45;
__m256i vacc2x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -93,23 +93,23 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
vacc2x01 = _mm256_add_epi32(vacc2x01, _mm256_madd_epi16(vxa2, vxb01));
- const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 16));
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
vacc2x23 = _mm256_add_epi32(vacc2x23, _mm256_madd_epi16(vxa2, vxb23));
- const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 32 * sizeof(int16_t)));
+ const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 32));
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
vacc2x45 = _mm256_add_epi32(vacc2x45, _mm256_madd_epi16(vxa2, vxb45));
- const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 48 * sizeof(int16_t)));
+ const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 48));
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
vacc2x67 = _mm256_add_epi32(vacc2x67, _mm256_madd_epi16(vxa2, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 64);
k += 8 * sizeof(int8_t);
}
@@ -134,7 +134,7 @@
__m256 vscaled2x01234567 = _mm256_cvtepi32_ps(vacc2x01234567);
const __m256 vscale01234567 = _mm256_load_ps(w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ w = (const void*) ((const float*) w + 8);
vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale01234567);
vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale01234567);
vscaled2x01234567 = _mm256_mul_ps(vscaled2x01234567, vscale01234567);
@@ -199,8 +199,8 @@
}
if (nc & 1) {
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
- *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
+ *c2 = (int8_t) _mm_extract_epi8(vout_lo, 8);
}
nc = 0;
diff --git a/src/qc8-igemm/gen/1x8c8-minmax-fp32-avx2.c b/src/qc8-igemm/gen/1x8c8-minmax-fp32-avx2.c
index 7a585c0..e111b16 100644
--- a/src/qc8-igemm/gen/1x8c8-minmax-fp32-avx2.c
+++ b/src/qc8-igemm/gen/1x8c8-minmax-fp32-avx2.c
@@ -46,18 +46,18 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t p = ks;
do {
@@ -77,20 +77,20 @@
const __m256i vxb01 = _mm256_cvtepi8_epi16(vb01);
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
@@ -107,7 +107,7 @@
__m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
const __m256 vscale01234567 = _mm256_load_ps(w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ w = (const void*) ((const float*) w + 8);
vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale01234567);
vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
diff --git a/src/qc8-igemm/gen/2x8c8-minmax-fp32-avx2.c b/src/qc8-igemm/gen/2x8c8-minmax-fp32-avx2.c
index c90bd00..5dd39af 100644
--- a/src/qc8-igemm/gen/2x8c8-minmax-fp32-avx2.c
+++ b/src/qc8-igemm/gen/2x8c8-minmax-fp32-avx2.c
@@ -50,22 +50,22 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
__m256i vacc1x45 = vacc0x45;
__m256i vacc1x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t p = ks;
do {
@@ -93,23 +93,23 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
@@ -131,7 +131,7 @@
__m256 vscaled1x01234567 = _mm256_cvtepi32_ps(vacc1x01234567);
const __m256 vscale01234567 = _mm256_load_ps(w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ w = (const void*) ((const float*) w + 8);
vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale01234567);
vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale01234567);
@@ -183,7 +183,7 @@
vout_hi = _mm_srli_epi32(vout_hi, 16);
}
if (nc & 1) {
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
}
diff --git a/src/qc8-igemm/gen/3x8c8-minmax-fp32-avx2.c b/src/qc8-igemm/gen/3x8c8-minmax-fp32-avx2.c
index 4030646..e3ccc08 100644
--- a/src/qc8-igemm/gen/3x8c8-minmax-fp32-avx2.c
+++ b/src/qc8-igemm/gen/3x8c8-minmax-fp32-avx2.c
@@ -54,16 +54,16 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
@@ -73,7 +73,7 @@
__m256i vacc2x23 = vacc0x23;
__m256i vacc2x45 = vacc0x45;
__m256i vacc2x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t p = ks;
do {
@@ -109,26 +109,26 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
vacc2x01 = _mm256_add_epi32(vacc2x01, _mm256_madd_epi16(vxa2, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
vacc2x23 = _mm256_add_epi32(vacc2x23, _mm256_madd_epi16(vxa2, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
vacc2x45 = _mm256_add_epi32(vacc2x45, _mm256_madd_epi16(vxa2, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
vacc2x67 = _mm256_add_epi32(vacc2x67, _mm256_madd_epi16(vxa2, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
@@ -155,7 +155,7 @@
__m256 vscaled2x01234567 = _mm256_cvtepi32_ps(vacc2x01234567);
const __m256 vscale01234567 = _mm256_load_ps(w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ w = (const void*) ((const float*) w + 8);
vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale01234567);
vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale01234567);
vscaled2x01234567 = _mm256_mul_ps(vscaled2x01234567, vscale01234567);
@@ -217,8 +217,8 @@
vout_hi = _mm_srli_epi32(vout_hi, 16);
}
if (nc & 1) {
- *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c2 = (int8_t) _mm_extract_epi8(vout_lo, 8);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
}
diff --git a/src/qs8-gemm/MRx8c8-avx2.c.in b/src/qs8-gemm/MRx8c8-avx2.c.in
index 647495f..f6287f1 100644
--- a/src/qs8-gemm/MRx8c8-avx2.c.in
+++ b/src/qs8-gemm/MRx8c8-avx2.c.in
@@ -4,7 +4,7 @@
// LICENSE file in the root directory of this source tree.
$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
-$assert DATATYPE in ["QC8", "QS8"]
+$assert DATATYPE in ["QC8", "QS8", "QU8"]
$assert DATATYPE != "QC8" or REQUANTIZATION == "FP32"
$assert VARIANT in ["LD128", "EXTENDED"]
$assert MR <= 4
@@ -19,15 +19,16 @@
$PARAMS_STRUCT = "avx2" if DATATYPE == "QC8" else REQUANTIZATION.lower() + "_avx2"
$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else ""
-$PARAMS_UNION = "xnn_qs8_minmax_params" if DATATYPE == "QC8" else "xnn_qs8_conv_minmax_params"
+$PARAMS_UNION = "xnn_qs8_minmax_params" if DATATYPE == "QC8" else "xnn_%s_conv_minmax_params" % DATATYPE.lower()
+$XINT8_T = "uint8_t" if DATATYPE == "QU8" else "int8_t"
void xnn_${DATATYPE.lower()}_gemm${GEMM_SUFFIX}_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x8c8__avx2(
size_t mr,
size_t nc,
size_t kc,
- const int8_t* restrict a,
+ const ${XINT8_T}* restrict a,
size_t a_stride,
const void* restrict w,
- int8_t* restrict c,
+ ${XINT8_T}* restrict c,
size_t cm_stride,
size_t cn_stride,
const union ${PARAMS_UNION} params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
@@ -36,17 +37,17 @@
assert(mr <= ${MR});
assert(nc != 0);
assert(kc != 0);
- assert(kc % sizeof(int8_t) == 0);
+ assert(kc % sizeof(${XINT8_T}) == 0);
assert(a != NULL);
assert(w != NULL);
assert(c != NULL);
kc = round_up_po2(kc, 8);
- const int8_t* a0 = a;
- int8_t* c0 = c;
+ const ${XINT8_T}* a0 = a;
+ ${XINT8_T}* c0 = c;
$for M in range(1, MR):
- const int8_t* a${M} = (const int8_t*) ((uintptr_t) a${M-1} + a_stride);
- int8_t* c${M} = (int8_t*) ((uintptr_t) c${M-1} + cm_stride);
+ const ${XINT8_T}* a${M} = (const ${XINT8_T}*) ((uintptr_t) a${M-1} + a_stride);
+ ${XINT8_T}* c${M} = (${XINT8_T}*) ((uintptr_t) c${M-1} + cm_stride);
$if M % 2 == 0:
if XNN_UNPREDICTABLE(mr <= ${M}) {
a${M} = a${M-1};
@@ -65,22 +66,27 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
$for N in range(2, 8, 2):
- const __m128i vbias0x${N} = _mm_loadu_si32((const void*) ((uintptr_t) w + ${N} * sizeof(int32_t)));
- const __m128i vbias0x${N+1} = _mm_loadu_si32((const void*) ((uintptr_t) w + ${N+1} * sizeof(int32_t)));
+ const __m128i vbias0x${N} = _mm_loadu_si32((const int32_t*) w + ${N});
+ const __m128i vbias0x${N+1} = _mm_loadu_si32((const int32_t*) w + ${N+1});
__m256i vacc0x${N}${N+1} = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x${N}), vbias0x${N+1}, 1);
$for M in range(1, MR):
$for N in range(0, 8, 2):
__m256i vacc${M}x${N}${N+1} = vacc0x${N}${N+1};
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
+ $if DATATYPE == "QU8":
+ const __m256i vb_zero_point = _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.kernel_zero_point);
while (k < kc) {
$for M in range(MR):
const __m128i va${M} = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a${M}));
- const __m256i vxa${M} = _mm256_cvtepi8_epi16(va${M});
+ $if DATATYPE == "QU8":
+ const __m256i vxa${M} = _mm256_cvtepu8_epi16(va${M});
+ $else:
+ const __m256i vxa${M} = _mm256_cvtepi8_epi16(va${M});
a${M} += 8;
$for N in range(0, 8, 2):
@@ -88,22 +94,25 @@
$if N == 0:
const __m256i vxb${N}${N+1} = _mm256_load_si256((const __m256i*) w);
$else:
- const __m256i vxb${N}${N+1} = _mm256_load_si256((const __m256i*) ((uintptr_t) w + ${N * 8} * sizeof(int16_t)));
+ const __m256i vxb${N}${N+1} = _mm256_load_si256((const __m256i*) ((const int16_t*) w + ${N * 8}));
$else:
$if N == 0:
const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) w);
$else:
- const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) ((uintptr_t) w + ${N * 8} * sizeof(int8_t)));
- const __m256i vxb${N}${N+1} = _mm256_cvtepi8_epi16(vb${N}${N+1});
+ const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) ((const ${XINT8_T}*) w + ${N * 8}));
+ $if DATATYPE == "QU8":
+ const __m256i vxb${N}${N+1} = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb${N}${N+1}), vb_zero_point);
+ $else:
+ const __m256i vxb${N}${N+1} = _mm256_cvtepi8_epi16(vb${N}${N+1});
$for M in range(MR):
vacc${M}x${N}${N+1} = _mm256_add_epi32(vacc${M}x${N}${N+1}, _mm256_madd_epi16(vxa${M}, vxb${N}${N+1}));
$if VARIANT == "EXTENDED":
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 64);
$else:
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
- k += 8 * sizeof(int8_t);
+ w = (const void*) ((const ${XINT8_T}*) w + 64);
+ k += 8 * sizeof(${XINT8_T});
}
$for M in range(MR):
@@ -156,7 +165,7 @@
$if DATATYPE == "QC8":
const __m256 vscale01234567 = _mm256_load_ps(w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ w = (const void*) ((const float*) w + 8);
$for M in range(MR):
vscaled${M}x01234567 = _mm256_mul_ps(vscaled${M}x01234567, vscale01234567);
$else:
@@ -174,13 +183,22 @@
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_permute4x64_epi64(vacc${M}${min(M+1, MR-1)}x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- $if MR > 2:
- __m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc${min(2, MR-1)}${min(3, MR-1)}x01234567);
- $else:
- __m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567);
+ $if DATATYPE == "QU8":
+ $if MR > 2:
+ __m256i vout = _mm256_packus_epi16(vacc0${min(1, MR-1)}x01234567, vacc${min(2, MR-1)}${min(3, MR-1)}x01234567);
+ $else:
+ __m256i vout = _mm256_packus_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_max));
+ vout = _mm256_max_epu8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_min));
+ vout = _mm256_min_epu8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_max));
+ $else:
+ $if MR > 2:
+ __m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc${min(2, MR-1)}${min(3, MR-1)}x01234567);
+ $else:
+ __m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
@@ -195,10 +213,10 @@
_mm_storeh_pi((__m64*) c3, _mm_castsi128_ps(vout_hi));
$for M in range(MR):
- c${M} = (int8_t*) ((uintptr_t) c${M} + cn_stride);
+ c${M} = (${XINT8_T}*) ((uintptr_t) c${M} + cn_stride);
$for M in range(MR):
- a${M} = (const int8_t*) ((uintptr_t) a${M} - kc);
+ a${M} = (const ${XINT8_T}*) ((uintptr_t) a${M} - kc);
nc -= 8;
} else {
@@ -233,13 +251,13 @@
vout_hi = _mm_srli_epi32(vout_hi, 16);
}
if (nc & 1) {
- *c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
+ *c0 = (${XINT8_T}) _mm_extract_epi8(vout_lo, 0);
$if MR > 1:
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c1 = (${XINT8_T}) _mm_extract_epi8(vout_hi, 0);
$if MR > 2:
- *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
+ *c2 = (${XINT8_T}) _mm_extract_epi8(vout_lo, 8);
$if MR > 3:
- *c3 = (uint8_t) _mm_extract_epi8(vout_hi, 8);
+ *c3 = (${XINT8_T}) _mm_extract_epi8(vout_hi, 8);
}
nc = 0;
diff --git a/src/qs8-gemm/gen/1x8c8-minmax-fp32-avx2.c b/src/qs8-gemm/gen/1x8c8-minmax-fp32-avx2.c
index 9c1b57c..0019358 100644
--- a/src/qs8-gemm/gen/1x8c8-minmax-fp32-avx2.c
+++ b/src/qs8-gemm/gen/1x8c8-minmax-fp32-avx2.c
@@ -43,18 +43,18 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -66,20 +66,20 @@
const __m256i vxb01 = _mm256_cvtepi8_epi16(vb01);
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-avx2.c
index e1761aa..50bd740 100644
--- a/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-avx2.c
@@ -43,18 +43,18 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -66,20 +66,20 @@
const __m256i vxb01 = _mm256_cvtepi8_epi16(vb01);
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x8c8-xw-minmax-fp32-avx2.c b/src/qs8-gemm/gen/1x8c8-xw-minmax-fp32-avx2.c
index 702c4ad..0996c79 100644
--- a/src/qs8-gemm/gen/1x8c8-xw-minmax-fp32-avx2.c
+++ b/src/qs8-gemm/gen/1x8c8-xw-minmax-fp32-avx2.c
@@ -43,18 +43,18 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -65,17 +65,17 @@
const __m256i vxb01 = _mm256_load_si256((const __m256i*) w);
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
- const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 16));
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
- const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 32 * sizeof(int16_t)));
+ const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 32));
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
- const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 48 * sizeof(int16_t)));
+ const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 48));
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 64);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x8c8-xw-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/1x8c8-xw-minmax-gemmlowp-avx2.c
index be4ec37..8b989f5 100644
--- a/src/qs8-gemm/gen/1x8c8-xw-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/1x8c8-xw-minmax-gemmlowp-avx2.c
@@ -43,18 +43,18 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -65,17 +65,17 @@
const __m256i vxb01 = _mm256_load_si256((const __m256i*) w);
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
- const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 16));
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
- const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 32 * sizeof(int16_t)));
+ const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 32));
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
- const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 48 * sizeof(int16_t)));
+ const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 48));
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 64);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x8c8-minmax-fp32-avx2.c b/src/qs8-gemm/gen/2x8c8-minmax-fp32-avx2.c
index d63d4f9..a074715 100644
--- a/src/qs8-gemm/gen/2x8c8-minmax-fp32-avx2.c
+++ b/src/qs8-gemm/gen/2x8c8-minmax-fp32-avx2.c
@@ -49,22 +49,22 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
__m256i vacc1x45 = vacc0x45;
__m256i vacc1x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -80,23 +80,23 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
@@ -169,7 +169,7 @@
}
if (nc & 1) {
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
}
nc = 0;
diff --git a/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-avx2.c
index 1e39921..d277d6f 100644
--- a/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-avx2.c
@@ -49,22 +49,22 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
__m256i vacc1x45 = vacc0x45;
__m256i vacc1x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -80,23 +80,23 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
@@ -192,7 +192,7 @@
}
if (nc & 1) {
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
}
nc = 0;
diff --git a/src/qs8-gemm/gen/2x8c8-xw-minmax-fp32-avx2.c b/src/qs8-gemm/gen/2x8c8-xw-minmax-fp32-avx2.c
index 6ce97ee..4d62e32 100644
--- a/src/qs8-gemm/gen/2x8c8-xw-minmax-fp32-avx2.c
+++ b/src/qs8-gemm/gen/2x8c8-xw-minmax-fp32-avx2.c
@@ -49,22 +49,22 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
__m256i vacc1x45 = vacc0x45;
__m256i vacc1x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -79,20 +79,20 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
- const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 16));
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
- const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 32 * sizeof(int16_t)));
+ const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 32));
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
- const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 48 * sizeof(int16_t)));
+ const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 48));
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 64);
k += 8 * sizeof(int8_t);
}
@@ -165,7 +165,7 @@
}
if (nc & 1) {
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
}
nc = 0;
diff --git a/src/qs8-gemm/gen/2x8c8-xw-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/2x8c8-xw-minmax-gemmlowp-avx2.c
index 810bae4..684ea57 100644
--- a/src/qs8-gemm/gen/2x8c8-xw-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/2x8c8-xw-minmax-gemmlowp-avx2.c
@@ -49,22 +49,22 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
__m256i vacc1x45 = vacc0x45;
__m256i vacc1x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -79,20 +79,20 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
- const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 16));
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
- const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 32 * sizeof(int16_t)));
+ const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 32));
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
- const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 48 * sizeof(int16_t)));
+ const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 48));
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 64);
k += 8 * sizeof(int8_t);
}
@@ -188,7 +188,7 @@
}
if (nc & 1) {
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
}
nc = 0;
diff --git a/src/qs8-gemm/gen/3x8c8-minmax-fp32-avx2.c b/src/qs8-gemm/gen/3x8c8-minmax-fp32-avx2.c
index 60b38a7..5904923 100644
--- a/src/qs8-gemm/gen/3x8c8-minmax-fp32-avx2.c
+++ b/src/qs8-gemm/gen/3x8c8-minmax-fp32-avx2.c
@@ -55,16 +55,16 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
@@ -74,7 +74,7 @@
__m256i vacc2x23 = vacc0x23;
__m256i vacc2x45 = vacc0x45;
__m256i vacc2x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -94,26 +94,26 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
vacc2x01 = _mm256_add_epi32(vacc2x01, _mm256_madd_epi16(vxa2, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
vacc2x23 = _mm256_add_epi32(vacc2x23, _mm256_madd_epi16(vxa2, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
vacc2x45 = _mm256_add_epi32(vacc2x45, _mm256_madd_epi16(vxa2, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
vacc2x67 = _mm256_add_epi32(vacc2x67, _mm256_madd_epi16(vxa2, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
@@ -202,8 +202,8 @@
}
if (nc & 1) {
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
- *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
+ *c2 = (int8_t) _mm_extract_epi8(vout_lo, 8);
}
nc = 0;
diff --git a/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-avx2.c
index b87cc62..5e4b270 100644
--- a/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/3x8c8-minmax-gemmlowp-avx2.c
@@ -55,16 +55,16 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
@@ -74,7 +74,7 @@
__m256i vacc2x23 = vacc0x23;
__m256i vacc2x45 = vacc0x45;
__m256i vacc2x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -94,26 +94,26 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
vacc2x01 = _mm256_add_epi32(vacc2x01, _mm256_madd_epi16(vxa2, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
vacc2x23 = _mm256_add_epi32(vacc2x23, _mm256_madd_epi16(vxa2, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
vacc2x45 = _mm256_add_epi32(vacc2x45, _mm256_madd_epi16(vxa2, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
vacc2x67 = _mm256_add_epi32(vacc2x67, _mm256_madd_epi16(vxa2, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
@@ -232,8 +232,8 @@
}
if (nc & 1) {
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
- *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
+ *c2 = (int8_t) _mm_extract_epi8(vout_lo, 8);
}
nc = 0;
diff --git a/src/qs8-gemm/gen/3x8c8-xw-minmax-fp32-avx2.c b/src/qs8-gemm/gen/3x8c8-xw-minmax-fp32-avx2.c
index 0166d53..065970e 100644
--- a/src/qs8-gemm/gen/3x8c8-xw-minmax-fp32-avx2.c
+++ b/src/qs8-gemm/gen/3x8c8-xw-minmax-fp32-avx2.c
@@ -55,16 +55,16 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
@@ -74,7 +74,7 @@
__m256i vacc2x23 = vacc0x23;
__m256i vacc2x45 = vacc0x45;
__m256i vacc2x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -93,23 +93,23 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
vacc2x01 = _mm256_add_epi32(vacc2x01, _mm256_madd_epi16(vxa2, vxb01));
- const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 16));
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
vacc2x23 = _mm256_add_epi32(vacc2x23, _mm256_madd_epi16(vxa2, vxb23));
- const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 32 * sizeof(int16_t)));
+ const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 32));
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
vacc2x45 = _mm256_add_epi32(vacc2x45, _mm256_madd_epi16(vxa2, vxb45));
- const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 48 * sizeof(int16_t)));
+ const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 48));
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
vacc2x67 = _mm256_add_epi32(vacc2x67, _mm256_madd_epi16(vxa2, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 64);
k += 8 * sizeof(int8_t);
}
@@ -198,8 +198,8 @@
}
if (nc & 1) {
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
- *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
+ *c2 = (int8_t) _mm_extract_epi8(vout_lo, 8);
}
nc = 0;
diff --git a/src/qs8-gemm/gen/3x8c8-xw-minmax-gemmlowp-avx2.c b/src/qs8-gemm/gen/3x8c8-xw-minmax-gemmlowp-avx2.c
index 0a4326c..019873f 100644
--- a/src/qs8-gemm/gen/3x8c8-xw-minmax-gemmlowp-avx2.c
+++ b/src/qs8-gemm/gen/3x8c8-xw-minmax-gemmlowp-avx2.c
@@ -55,16 +55,16 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
@@ -74,7 +74,7 @@
__m256i vacc2x23 = vacc0x23;
__m256i vacc2x45 = vacc0x45;
__m256i vacc2x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t k = 0;
while (k < kc) {
@@ -93,23 +93,23 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
vacc2x01 = _mm256_add_epi32(vacc2x01, _mm256_madd_epi16(vxa2, vxb01));
- const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m256i vxb23 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 16));
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
vacc2x23 = _mm256_add_epi32(vacc2x23, _mm256_madd_epi16(vxa2, vxb23));
- const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 32 * sizeof(int16_t)));
+ const __m256i vxb45 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 32));
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
vacc2x45 = _mm256_add_epi32(vacc2x45, _mm256_madd_epi16(vxa2, vxb45));
- const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((uintptr_t) w + 48 * sizeof(int16_t)));
+ const __m256i vxb67 = _mm256_load_si256((const __m256i*) ((const int16_t*) w + 48));
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
vacc2x67 = _mm256_add_epi32(vacc2x67, _mm256_madd_epi16(vxa2, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 64);
k += 8 * sizeof(int8_t);
}
@@ -228,8 +228,8 @@
}
if (nc & 1) {
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
- *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
+ *c2 = (int8_t) _mm_extract_epi8(vout_lo, 8);
}
nc = 0;
diff --git a/src/qs8-igemm/MRx8c8-avx2.c.in b/src/qs8-igemm/MRx8c8-avx2.c.in
index 91ed141..69d03ba 100644
--- a/src/qs8-igemm/MRx8c8-avx2.c.in
+++ b/src/qs8-igemm/MRx8c8-avx2.c.in
@@ -4,7 +4,7 @@
// LICENSE file in the root directory of this source tree.
$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
-$assert DATATYPE in ["QC8", "QS8"]
+$assert DATATYPE in ["QC8", "QS8", "QU8"]
$assert DATATYPE != "QC8" or REQUANTIZATION == "FP32"
$assert MR <= 4
#include <assert.h>
@@ -17,19 +17,20 @@
$PARAMS_STRUCT = "avx2" if DATATYPE == "QC8" else REQUANTIZATION.lower() + "_avx2"
-$PARAMS_UNION = "xnn_qs8_minmax_params" if DATATYPE == "QC8" else "xnn_qs8_conv_minmax_params"
+$PARAMS_UNION = "xnn_qs8_minmax_params" if DATATYPE == "QC8" else "xnn_%s_conv_minmax_params" % DATATYPE.lower()
+$XINT8_T = "uint8_t" if DATATYPE == "QU8" else "int8_t"
void xnn_${DATATYPE.lower()}_igemm_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x8c8__avx2(
size_t mr,
size_t nc,
size_t kc,
size_t ks,
- const int8_t** restrict a,
+ const ${XINT8_T}** restrict a,
const void* restrict w,
- int8_t* restrict c,
+ ${XINT8_T}* restrict c,
size_t cm_stride,
size_t cn_stride,
size_t a_offset,
- const int8_t* zero,
+ const ${XINT8_T}* zero,
const union ${PARAMS_UNION} params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
{
assert(mr != 0);
@@ -38,15 +39,15 @@
assert(kc != 0);
assert(ks != 0);
assert(ks % (${MR} * sizeof(void*)) == 0);
- assert(a_offset % sizeof(int8_t) == 0);
+ assert(a_offset % sizeof(${XINT8_T}) == 0);
assert(a != NULL);
assert(w != NULL);
assert(c != NULL);
kc = round_up_po2(kc, 8);
- int8_t* c0 = c;
+ ${XINT8_T}* c0 = c;
$for M in range(1, MR):
- int8_t* c${M} = (int8_t*) ((uintptr_t) c${M-1} + cm_stride);
+ ${XINT8_T}* c${M} = (${XINT8_T}*) ((uintptr_t) c${M-1} + cm_stride);
$if M % 2 == 0:
if XNN_UNPREDICTABLE(mr <= ${M}) {
c${M} = c${M-1};
@@ -62,23 +63,25 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
$for N in range(2, 8, 2):
- const __m128i vbias0x${N} = _mm_loadu_si32((const void*) ((uintptr_t) w + ${N} * sizeof(int32_t)));
- const __m128i vbias0x${N+1} = _mm_loadu_si32((const void*) ((uintptr_t) w + ${N+1} * sizeof(int32_t)));
+ const __m128i vbias0x${N} = _mm_loadu_si32((const int32_t*) w + ${N});
+ const __m128i vbias0x${N+1} = _mm_loadu_si32((const int32_t*) w + ${N+1});
__m256i vacc0x${N}${N+1} = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x${N}), vbias0x${N+1}, 1);
$for M in range(1, MR):
$for N in range(0, 8, 2):
__m256i vacc${M}x${N}${N+1} = vacc0x${N}${N+1};
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t p = ks;
+ $if DATATYPE == "QU8":
+ const __m256i vb_zero_point = _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.kernel_zero_point);
do {
$for M in range(MR):
- const int8_t* restrict a${M} = a[${M}];
+ const ${XINT8_T}* restrict a${M} = a[${M}];
if XNN_UNPREDICTABLE(a${M} != zero) {
- a${M} = (const int8_t*) ((uintptr_t) a${M} + a_offset);
+ a${M} = (const ${XINT8_T}*) ((uintptr_t) a${M} + a_offset);
}
a += ${MR};
@@ -86,21 +89,27 @@
while (k < kc) {
$for M in range(MR):
const __m128i va${M} = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a${M}));
- const __m256i vxa${M} = _mm256_cvtepi8_epi16(va${M});
+ $if DATATYPE == "QU8":
+ const __m256i vxa${M} = _mm256_cvtepu8_epi16(va${M});
+ $else:
+ const __m256i vxa${M} = _mm256_cvtepi8_epi16(va${M});
a${M} += 8;
$for N in range(0, 8, 2):
$if N == 0:
const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) w);
$else:
- const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) ((uintptr_t) w + ${N * 8} * sizeof(int8_t)));
- const __m256i vxb${N}${N+1} = _mm256_cvtepi8_epi16(vb${N}${N+1});
+ const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) ((const ${XINT8_T}*) w + ${N * 8}));
+ $if DATATYPE == "QU8":
+ const __m256i vxb${N}${N+1} = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb${N}${N+1}), vb_zero_point);
+ $else:
+ const __m256i vxb${N}${N+1} = _mm256_cvtepi8_epi16(vb${N}${N+1});
$for M in range(MR):
vacc${M}x${N}${N+1} = _mm256_add_epi32(vacc${M}x${N}${N+1}, _mm256_madd_epi16(vxa${M}, vxb${N}${N+1}));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
- k += 8 * sizeof(int8_t);
+ w = (const void*) ((const ${XINT8_T}*) w + 64);
+ k += 8 * sizeof(${XINT8_T});
}
p -= ${MR} * sizeof(void*);
} while (p != 0);
@@ -155,7 +164,7 @@
$if DATATYPE == "QC8":
const __m256 vscale01234567 = _mm256_load_ps(w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ w = (const void*) ((const float*) w + 8);
$for M in range(MR):
vscaled${M}x01234567 = _mm256_mul_ps(vscaled${M}x01234567, vscale01234567);
$else:
@@ -173,13 +182,22 @@
$for M in range(0, MR, 2):
vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_permute4x64_epi64(vacc${M}${min(M+1, MR-1)}x01234567, _MM_SHUFFLE(3, 1, 2, 0));
- $if MR > 2:
- __m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc${min(2, MR-1)}${min(3, MR-1)}x01234567);
- $else:
- __m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567);
+ $if DATATYPE == "QU8":
+ $if MR > 2:
+ __m256i vout = _mm256_packus_epi16(vacc0${min(1, MR-1)}x01234567, vacc${min(2, MR-1)}${min(3, MR-1)}x01234567);
+ $else:
+ __m256i vout = _mm256_packus_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567);
- vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_min));
- vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_max));
+ vout = _mm256_max_epu8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_min));
+ vout = _mm256_min_epu8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_max));
+ $else:
+ $if MR > 2:
+ __m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc${min(2, MR-1)}${min(3, MR-1)}x01234567);
+ $else:
+ __m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567);
+
+ vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_min));
+ vout = _mm256_min_epi8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_max));
__m128i vout_lo = _mm256_castsi256_si128(vout);
__m128i vout_hi = _mm256_extracti128_si256(vout, 1);
@@ -194,9 +212,9 @@
_mm_storel_epi64((__m128i*) c0, vout_lo);
$for M in reversed(range(MR)):
- c${M} = (int8_t*) ((uintptr_t) c${M} + cn_stride);
+ c${M} = (${XINT8_T}*) ((uintptr_t) c${M} + cn_stride);
- a = (const int8_t**restrict) ((uintptr_t) a - ks);
+ a = (const ${XINT8_T}**restrict) ((uintptr_t) a - ks);
nc -= 8;
} else {
@@ -232,12 +250,12 @@
}
if (nc & 1) {
$if MR > 3:
- *c3 = (uint8_t) _mm_extract_epi8(vout_hi, 8);
+ *c3 = (${XINT8_T}) _mm_extract_epi8(vout_hi, 8);
$if MR > 2:
- *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
+ *c2 = (${XINT8_T}) _mm_extract_epi8(vout_lo, 8);
$if MR > 1:
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
- *c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
+ *c1 = (${XINT8_T}) _mm_extract_epi8(vout_hi, 0);
+ *c0 = (${XINT8_T}) _mm_extract_epi8(vout_lo, 0);
}
nc = 0;
diff --git a/src/qs8-igemm/gen/1x8c8-minmax-fp32-avx2.c b/src/qs8-igemm/gen/1x8c8-minmax-fp32-avx2.c
index dc3a7a3..ca4e5a4 100644
--- a/src/qs8-igemm/gen/1x8c8-minmax-fp32-avx2.c
+++ b/src/qs8-igemm/gen/1x8c8-minmax-fp32-avx2.c
@@ -46,18 +46,18 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t p = ks;
do {
@@ -77,20 +77,20 @@
const __m256i vxb01 = _mm256_cvtepi8_epi16(vb01);
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-avx2.c b/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-avx2.c
index 34aedf9..c40c7b4 100644
--- a/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-avx2.c
+++ b/src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-avx2.c
@@ -46,18 +46,18 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t p = ks;
do {
@@ -77,20 +77,20 @@
const __m256i vxb01 = _mm256_cvtepi8_epi16(vb01);
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x8c8-minmax-fp32-avx2.c b/src/qs8-igemm/gen/2x8c8-minmax-fp32-avx2.c
index ab6220d..9e1c011 100644
--- a/src/qs8-igemm/gen/2x8c8-minmax-fp32-avx2.c
+++ b/src/qs8-igemm/gen/2x8c8-minmax-fp32-avx2.c
@@ -50,22 +50,22 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
__m256i vacc1x45 = vacc0x45;
__m256i vacc1x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t p = ks;
do {
@@ -93,23 +93,23 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
@@ -182,7 +182,7 @@
vout_hi = _mm_srli_epi32(vout_hi, 16);
}
if (nc & 1) {
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
}
diff --git a/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-avx2.c b/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-avx2.c
index a5c416d..bf8d9af 100644
--- a/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-avx2.c
+++ b/src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-avx2.c
@@ -50,22 +50,22 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
__m256i vacc1x45 = vacc0x45;
__m256i vacc1x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t p = ks;
do {
@@ -93,23 +93,23 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
@@ -205,7 +205,7 @@
vout_hi = _mm_srli_epi32(vout_hi, 16);
}
if (nc & 1) {
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
}
diff --git a/src/qs8-igemm/gen/3x8c8-minmax-fp32-avx2.c b/src/qs8-igemm/gen/3x8c8-minmax-fp32-avx2.c
index ecd6239..b2faab7 100644
--- a/src/qs8-igemm/gen/3x8c8-minmax-fp32-avx2.c
+++ b/src/qs8-igemm/gen/3x8c8-minmax-fp32-avx2.c
@@ -54,16 +54,16 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
@@ -73,7 +73,7 @@
__m256i vacc2x23 = vacc0x23;
__m256i vacc2x45 = vacc0x45;
__m256i vacc2x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t p = ks;
do {
@@ -109,26 +109,26 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
vacc2x01 = _mm256_add_epi32(vacc2x01, _mm256_madd_epi16(vxa2, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
vacc2x23 = _mm256_add_epi32(vacc2x23, _mm256_madd_epi16(vxa2, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
vacc2x45 = _mm256_add_epi32(vacc2x45, _mm256_madd_epi16(vxa2, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
vacc2x67 = _mm256_add_epi32(vacc2x67, _mm256_madd_epi16(vxa2, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
@@ -216,8 +216,8 @@
vout_hi = _mm_srli_epi32(vout_hi, 16);
}
if (nc & 1) {
- *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c2 = (int8_t) _mm_extract_epi8(vout_lo, 8);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
}
diff --git a/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-avx2.c b/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-avx2.c
index 32d5ac2..6214b27 100644
--- a/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-avx2.c
+++ b/src/qs8-igemm/gen/3x8c8-minmax-gemmlowp-avx2.c
@@ -54,16 +54,16 @@
do {
const __m128i vbias0x0 = _mm_loadu_si32(w);
- const __m128i vbias0x1 = _mm_loadu_si32((const void*) ((uintptr_t) w + sizeof(int32_t)));
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
__m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
- const __m128i vbias0x2 = _mm_loadu_si32((const void*) ((uintptr_t) w + 2 * sizeof(int32_t)));
- const __m128i vbias0x3 = _mm_loadu_si32((const void*) ((uintptr_t) w + 3 * sizeof(int32_t)));
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
__m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
- const __m128i vbias0x4 = _mm_loadu_si32((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- const __m128i vbias0x5 = _mm_loadu_si32((const void*) ((uintptr_t) w + 5 * sizeof(int32_t)));
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
__m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
- const __m128i vbias0x6 = _mm_loadu_si32((const void*) ((uintptr_t) w + 6 * sizeof(int32_t)));
- const __m128i vbias0x7 = _mm_loadu_si32((const void*) ((uintptr_t) w + 7 * sizeof(int32_t)));
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
__m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
__m256i vacc1x01 = vacc0x01;
__m256i vacc1x23 = vacc0x23;
@@ -73,7 +73,7 @@
__m256i vacc2x23 = vacc0x23;
__m256i vacc2x45 = vacc0x45;
__m256i vacc2x67 = vacc0x67;
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
size_t p = ks;
do {
@@ -109,26 +109,26 @@
vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
vacc2x01 = _mm256_add_epi32(vacc2x01, _mm256_madd_epi16(vxa2, vxb01));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m256i vxb23 = _mm256_cvtepi8_epi16(vb23);
vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
vacc2x23 = _mm256_add_epi32(vacc2x23, _mm256_madd_epi16(vxa2, vxb23));
- const __m128i vb45 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 32));
const __m256i vxb45 = _mm256_cvtepi8_epi16(vb45);
vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
vacc2x45 = _mm256_add_epi32(vacc2x45, _mm256_madd_epi16(vxa2, vxb45));
- const __m128i vb67 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 48));
const __m256i vxb67 = _mm256_cvtepi8_epi16(vb67);
vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
vacc2x67 = _mm256_add_epi32(vacc2x67, _mm256_madd_epi16(vxa2, vxb67));
- w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 64);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
@@ -246,8 +246,8 @@
vout_hi = _mm_srli_epi32(vout_hi, 16);
}
if (nc & 1) {
- *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
- *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c2 = (int8_t) _mm_extract_epi8(vout_lo, 8);
+ *c1 = (int8_t) _mm_extract_epi8(vout_hi, 0);
*c0 = (int8_t) _mm_extract_epi8(vout_lo, 0);
}
diff --git a/src/qu8-gemm/gen/1x8c8-minmax-fp32-avx2.c b/src/qu8-gemm/gen/1x8c8-minmax-fp32-avx2.c
new file mode 100644
index 0000000..e554acd
--- /dev/null
+++ b/src/qu8-gemm/gen/1x8c8-minmax-fp32-avx2.c
@@ -0,0 +1,147 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx8c8-avx2.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const uint8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ uint8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qu8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(uint8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const uint8_t* a0 = a;
+ uint8_t* c0 = c;
+
+ do {
+ const __m128i vbias0x0 = _mm_loadu_si32(w);
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
+ __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
+ __m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
+ __m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
+ __m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
+ w = (const void*) ((const int32_t*) w + 8);
+
+ size_t k = 0;
+ const __m256i vb_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.kernel_zero_point);
+ while (k < kc) {
+ const __m128i va0 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a0));
+ const __m256i vxa0 = _mm256_cvtepu8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m256i vxb01 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb01), vb_zero_point);
+
+ vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 16));
+ const __m256i vxb23 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb23), vb_zero_point);
+
+ vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 32));
+ const __m256i vxb45 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb45), vb_zero_point);
+
+ vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 48));
+ const __m256i vxb67 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb67), vb_zero_point);
+
+ vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
+
+ w = (const void*) ((const uint8_t*) w + 64);
+ k += 8 * sizeof(uint8_t);
+ }
+
+ const __m256i vacc0x0213 = _mm256_hadd_epi32(vacc0x01, vacc0x23);
+ const __m256i vacc0x4657 = _mm256_hadd_epi32(vacc0x45, vacc0x67);
+
+ const __m256i vacc0x02461357 = _mm256_hadd_epi32(vacc0x0213, vacc0x4657);
+
+ const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+ __m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
+
+ __m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale);
+
+ vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vacc00x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc0x01234567), voutput_zero_point);
+
+ vacc00x01234567 = _mm256_permute4x64_epi64(vacc00x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+
+ __m256i vout = _mm256_packus_epi16(vacc00x01234567, vacc00x01234567);
+
+ vout = _mm256_max_epu8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_min));
+ vout = _mm256_min_epu8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_max));
+
+ __m128i vout_lo = _mm256_castsi256_si128(vout);
+ __m128i vout_hi = _mm256_extracti128_si256(vout, 1);
+
+ if (nc >= 8) {
+ _mm_storel_epi64((__m128i*) c0, vout_lo);
+
+ c0 = (uint8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a0 = (const uint8_t*) ((uintptr_t) a0 - kc);
+
+ nc -= 8;
+ } else {
+ if (nc & 4) {
+ _mm_storeu_si32(c0, vout_lo);
+
+ c0 += 4;
+
+ vout_lo = _mm_srli_epi64(vout_lo, 32);
+ vout_hi = _mm_srli_epi64(vout_hi, 32);
+ }
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0);
+
+ c0 += 2;
+
+ vout_lo = _mm_srli_epi32(vout_lo, 16);
+ vout_hi = _mm_srli_epi32(vout_hi, 16);
+ }
+ if (nc & 1) {
+ *c0 = (uint8_t) _mm_extract_epi8(vout_lo, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qu8-gemm/gen/2x8c8-minmax-fp32-avx2.c b/src/qu8-gemm/gen/2x8c8-minmax-fp32-avx2.c
new file mode 100644
index 0000000..6519621
--- /dev/null
+++ b/src/qu8-gemm/gen/2x8c8-minmax-fp32-avx2.c
@@ -0,0 +1,179 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx8c8-avx2.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const uint8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ uint8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qu8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(uint8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const uint8_t* a0 = a;
+ uint8_t* c0 = c;
+ const uint8_t* a1 = (const uint8_t*) ((uintptr_t) a0 + a_stride);
+ uint8_t* c1 = (uint8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+
+ do {
+ const __m128i vbias0x0 = _mm_loadu_si32(w);
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
+ __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
+ __m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
+ __m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
+ __m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
+ __m256i vacc1x01 = vacc0x01;
+ __m256i vacc1x23 = vacc0x23;
+ __m256i vacc1x45 = vacc0x45;
+ __m256i vacc1x67 = vacc0x67;
+ w = (const void*) ((const int32_t*) w + 8);
+
+ size_t k = 0;
+ const __m256i vb_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.kernel_zero_point);
+ while (k < kc) {
+ const __m128i va0 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a0));
+ const __m256i vxa0 = _mm256_cvtepu8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a1));
+ const __m256i vxa1 = _mm256_cvtepu8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m256i vxb01 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb01), vb_zero_point);
+
+ vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
+ vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 16));
+ const __m256i vxb23 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb23), vb_zero_point);
+
+ vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
+ vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 32));
+ const __m256i vxb45 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb45), vb_zero_point);
+
+ vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
+ vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 48));
+ const __m256i vxb67 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb67), vb_zero_point);
+
+ vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
+ vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
+
+ w = (const void*) ((const uint8_t*) w + 64);
+ k += 8 * sizeof(uint8_t);
+ }
+
+ const __m256i vacc0x0213 = _mm256_hadd_epi32(vacc0x01, vacc0x23);
+ const __m256i vacc0x4657 = _mm256_hadd_epi32(vacc0x45, vacc0x67);
+ const __m256i vacc1x0213 = _mm256_hadd_epi32(vacc1x01, vacc1x23);
+ const __m256i vacc1x4657 = _mm256_hadd_epi32(vacc1x45, vacc1x67);
+
+ const __m256i vacc0x02461357 = _mm256_hadd_epi32(vacc0x0213, vacc0x4657);
+ const __m256i vacc1x02461357 = _mm256_hadd_epi32(vacc1x0213, vacc1x4657);
+
+ const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+ __m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
+ __m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
+
+ __m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
+ __m256 vscaled1x01234567 = _mm256_cvtepi32_ps(vacc1x01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale);
+ vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale);
+
+ vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
+ vacc1x01234567 = _mm256_cvtps_epi32(vscaled1x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
+
+ vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+
+ __m256i vout = _mm256_packus_epi16(vacc01x01234567, vacc01x01234567);
+
+ vout = _mm256_max_epu8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_min));
+ vout = _mm256_min_epu8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_max));
+
+ __m128i vout_lo = _mm256_castsi256_si128(vout);
+ __m128i vout_hi = _mm256_extracti128_si256(vout, 1);
+
+ if (nc >= 8) {
+ _mm_storel_epi64((__m128i*) c0, vout_lo);
+ _mm_storel_epi64((__m128i*) c1, vout_hi);
+
+ c0 = (uint8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (uint8_t*) ((uintptr_t) c1 + cn_stride);
+
+ a0 = (const uint8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const uint8_t*) ((uintptr_t) a1 - kc);
+
+ nc -= 8;
+ } else {
+ if (nc & 4) {
+ _mm_storeu_si32(c0, vout_lo);
+ _mm_storeu_si32(c1, vout_hi);
+
+ c0 += 4;
+ c1 += 4;
+
+ vout_lo = _mm_srli_epi64(vout_lo, 32);
+ vout_hi = _mm_srli_epi64(vout_hi, 32);
+ }
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0);
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout_hi, 0);
+
+ c0 += 2;
+ c1 += 2;
+
+ vout_lo = _mm_srli_epi32(vout_lo, 16);
+ vout_hi = _mm_srli_epi32(vout_hi, 16);
+ }
+ if (nc & 1) {
+ *c0 = (uint8_t) _mm_extract_epi8(vout_lo, 0);
+ *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qu8-gemm/gen/3x8c8-minmax-fp32-avx2.c b/src/qu8-gemm/gen/3x8c8-minmax-fp32-avx2.c
new file mode 100644
index 0000000..d47febd
--- /dev/null
+++ b/src/qu8-gemm/gen/3x8c8-minmax-fp32-avx2.c
@@ -0,0 +1,213 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/MRx8c8-avx2.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/gemm.h>
+#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ const uint8_t* restrict a,
+ size_t a_stride,
+ const void* restrict w,
+ uint8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ const union xnn_qu8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(kc % sizeof(uint8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ const uint8_t* a0 = a;
+ uint8_t* c0 = c;
+ const uint8_t* a1 = (const uint8_t*) ((uintptr_t) a0 + a_stride);
+ uint8_t* c1 = (uint8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ a1 = a0;
+ c1 = c0;
+ }
+ const uint8_t* a2 = (const uint8_t*) ((uintptr_t) a1 + a_stride);
+ uint8_t* c2 = (uint8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ a2 = a1;
+ c2 = c1;
+ }
+
+ do {
+ const __m128i vbias0x0 = _mm_loadu_si32(w);
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
+ __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
+ __m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
+ __m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
+ __m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
+ __m256i vacc1x01 = vacc0x01;
+ __m256i vacc1x23 = vacc0x23;
+ __m256i vacc1x45 = vacc0x45;
+ __m256i vacc1x67 = vacc0x67;
+ __m256i vacc2x01 = vacc0x01;
+ __m256i vacc2x23 = vacc0x23;
+ __m256i vacc2x45 = vacc0x45;
+ __m256i vacc2x67 = vacc0x67;
+ w = (const void*) ((const int32_t*) w + 8);
+
+ size_t k = 0;
+ const __m256i vb_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.kernel_zero_point);
+ while (k < kc) {
+ const __m128i va0 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a0));
+ const __m256i vxa0 = _mm256_cvtepu8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a1));
+ const __m256i vxa1 = _mm256_cvtepu8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a2));
+ const __m256i vxa2 = _mm256_cvtepu8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m256i vxb01 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb01), vb_zero_point);
+
+ vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
+ vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
+ vacc2x01 = _mm256_add_epi32(vacc2x01, _mm256_madd_epi16(vxa2, vxb01));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 16));
+ const __m256i vxb23 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb23), vb_zero_point);
+
+ vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
+ vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
+ vacc2x23 = _mm256_add_epi32(vacc2x23, _mm256_madd_epi16(vxa2, vxb23));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 32));
+ const __m256i vxb45 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb45), vb_zero_point);
+
+ vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
+ vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
+ vacc2x45 = _mm256_add_epi32(vacc2x45, _mm256_madd_epi16(vxa2, vxb45));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 48));
+ const __m256i vxb67 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb67), vb_zero_point);
+
+ vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
+ vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
+ vacc2x67 = _mm256_add_epi32(vacc2x67, _mm256_madd_epi16(vxa2, vxb67));
+
+ w = (const void*) ((const uint8_t*) w + 64);
+ k += 8 * sizeof(uint8_t);
+ }
+
+ const __m256i vacc0x0213 = _mm256_hadd_epi32(vacc0x01, vacc0x23);
+ const __m256i vacc0x4657 = _mm256_hadd_epi32(vacc0x45, vacc0x67);
+ const __m256i vacc1x0213 = _mm256_hadd_epi32(vacc1x01, vacc1x23);
+ const __m256i vacc1x4657 = _mm256_hadd_epi32(vacc1x45, vacc1x67);
+ const __m256i vacc2x0213 = _mm256_hadd_epi32(vacc2x01, vacc2x23);
+ const __m256i vacc2x4657 = _mm256_hadd_epi32(vacc2x45, vacc2x67);
+
+ const __m256i vacc0x02461357 = _mm256_hadd_epi32(vacc0x0213, vacc0x4657);
+ const __m256i vacc1x02461357 = _mm256_hadd_epi32(vacc1x0213, vacc1x4657);
+ const __m256i vacc2x02461357 = _mm256_hadd_epi32(vacc2x0213, vacc2x4657);
+
+ const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+ __m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
+ __m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
+ __m256i vacc2x01234567 = _mm256_permutevar8x32_epi32(vacc2x02461357, vpermute_mask);
+
+ __m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
+ __m256 vscaled1x01234567 = _mm256_cvtepi32_ps(vacc1x01234567);
+ __m256 vscaled2x01234567 = _mm256_cvtepi32_ps(vacc2x01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale);
+ vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale);
+ vscaled2x01234567 = _mm256_mul_ps(vscaled2x01234567, vscale);
+
+ vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
+ vacc1x01234567 = _mm256_cvtps_epi32(vscaled1x01234567);
+ vacc2x01234567 = _mm256_cvtps_epi32(vscaled2x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
+ __m256i vacc22x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc2x01234567, vacc2x01234567), voutput_zero_point);
+
+ vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+ vacc22x01234567 = _mm256_permute4x64_epi64(vacc22x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+
+ __m256i vout = _mm256_packus_epi16(vacc01x01234567, vacc22x01234567);
+
+ vout = _mm256_max_epu8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_min));
+ vout = _mm256_min_epu8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_max));
+
+ __m128i vout_lo = _mm256_castsi256_si128(vout);
+ __m128i vout_hi = _mm256_extracti128_si256(vout, 1);
+
+ if (nc >= 8) {
+ _mm_storel_epi64((__m128i*) c0, vout_lo);
+ _mm_storel_epi64((__m128i*) c1, vout_hi);
+ _mm_storeh_pi((__m64*) c2, _mm_castsi128_ps(vout_lo));
+
+ c0 = (uint8_t*) ((uintptr_t) c0 + cn_stride);
+ c1 = (uint8_t*) ((uintptr_t) c1 + cn_stride);
+ c2 = (uint8_t*) ((uintptr_t) c2 + cn_stride);
+
+ a0 = (const uint8_t*) ((uintptr_t) a0 - kc);
+ a1 = (const uint8_t*) ((uintptr_t) a1 - kc);
+ a2 = (const uint8_t*) ((uintptr_t) a2 - kc);
+
+ nc -= 8;
+ } else {
+ if (nc & 4) {
+ _mm_storeu_si32(c0, vout_lo);
+ _mm_storeu_si32(c1, vout_hi);
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout_lo, 2);
+
+ c0 += 4;
+ c1 += 4;
+ c2 += 4;
+
+ vout_lo = _mm_srli_epi64(vout_lo, 32);
+ vout_hi = _mm_srli_epi64(vout_hi, 32);
+ }
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0);
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout_hi, 0);
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout_lo, 4);
+
+ c0 += 2;
+ c1 += 2;
+ c2 += 2;
+
+ vout_lo = _mm_srli_epi32(vout_lo, 16);
+ vout_hi = _mm_srli_epi32(vout_hi, 16);
+ }
+ if (nc & 1) {
+ *c0 = (uint8_t) _mm_extract_epi8(vout_lo, 0);
+ *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qu8-igemm/gen/1x8c8-minmax-fp32-avx2.c b/src/qu8-igemm/gen/1x8c8-minmax-fp32-avx2.c
new file mode 100644
index 0000000..ea92339
--- /dev/null
+++ b/src/qu8-igemm/gen/1x8c8-minmax-fp32-avx2.c
@@ -0,0 +1,160 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx8c8-avx2.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const uint8_t** restrict a,
+ const void* restrict w,
+ uint8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const uint8_t* zero,
+ const union xnn_qu8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 1);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (1 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(uint8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ uint8_t* c0 = c;
+
+ do {
+ const __m128i vbias0x0 = _mm_loadu_si32(w);
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
+ __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
+ __m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
+ __m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
+ __m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
+ w = (const void*) ((const int32_t*) w + 8);
+
+ size_t p = ks;
+ const __m256i vb_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.kernel_zero_point);
+ do {
+ const uint8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const uint8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ a += 1;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a0));
+ const __m256i vxa0 = _mm256_cvtepu8_epi16(va0);
+ a0 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m256i vxb01 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb01), vb_zero_point);
+
+ vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 16));
+ const __m256i vxb23 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb23), vb_zero_point);
+
+ vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 32));
+ const __m256i vxb45 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb45), vb_zero_point);
+
+ vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 48));
+ const __m256i vxb67 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb67), vb_zero_point);
+
+ vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
+
+ w = (const void*) ((const uint8_t*) w + 64);
+ k += 8 * sizeof(uint8_t);
+ }
+ p -= 1 * sizeof(void*);
+ } while (p != 0);
+
+ const __m256i vacc0x0213 = _mm256_hadd_epi32(vacc0x01, vacc0x23);
+ const __m256i vacc0x4657 = _mm256_hadd_epi32(vacc0x45, vacc0x67);
+
+ const __m256i vacc0x02461357 = _mm256_hadd_epi32(vacc0x0213, vacc0x4657);
+
+ const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+ __m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
+
+ __m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale);
+
+ vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vacc00x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc0x01234567), voutput_zero_point);
+
+ vacc00x01234567 = _mm256_permute4x64_epi64(vacc00x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+
+ __m256i vout = _mm256_packus_epi16(vacc00x01234567, vacc00x01234567);
+
+ vout = _mm256_max_epu8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_min));
+ vout = _mm256_min_epu8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_max));
+
+ __m128i vout_lo = _mm256_castsi256_si128(vout);
+ __m128i vout_hi = _mm256_extracti128_si256(vout, 1);
+
+ if (nc >= 8) {
+ _mm_storel_epi64((__m128i*) c0, vout_lo);
+
+ c0 = (uint8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const uint8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 8;
+ } else {
+ if (nc & 4) {
+ _mm_storeu_si32(c0, vout_lo);
+
+ c0 += 4;
+
+ vout_lo = _mm_srli_epi64(vout_lo, 32);
+ vout_hi = _mm_srli_epi64(vout_hi, 32);
+ }
+ if (nc & 2) {
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0);
+
+ c0 += 2;
+
+ vout_lo = _mm_srli_epi32(vout_lo, 16);
+ vout_hi = _mm_srli_epi32(vout_hi, 16);
+ }
+ if (nc & 1) {
+ *c0 = (uint8_t) _mm_extract_epi8(vout_lo, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qu8-igemm/gen/2x8c8-minmax-fp32-avx2.c b/src/qu8-igemm/gen/2x8c8-minmax-fp32-avx2.c
new file mode 100644
index 0000000..b6679e3
--- /dev/null
+++ b/src/qu8-igemm/gen/2x8c8-minmax-fp32-avx2.c
@@ -0,0 +1,193 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx8c8-avx2.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const uint8_t** restrict a,
+ const void* restrict w,
+ uint8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const uint8_t* zero,
+ const union xnn_qu8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 2);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (2 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(uint8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ uint8_t* c0 = c;
+ uint8_t* c1 = (uint8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr != 2) {
+ c1 = c0;
+ }
+
+ do {
+ const __m128i vbias0x0 = _mm_loadu_si32(w);
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
+ __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
+ __m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
+ __m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
+ __m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
+ __m256i vacc1x01 = vacc0x01;
+ __m256i vacc1x23 = vacc0x23;
+ __m256i vacc1x45 = vacc0x45;
+ __m256i vacc1x67 = vacc0x67;
+ w = (const void*) ((const int32_t*) w + 8);
+
+ size_t p = ks;
+ const __m256i vb_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.kernel_zero_point);
+ do {
+ const uint8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const uint8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const uint8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const uint8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ a += 2;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a0));
+ const __m256i vxa0 = _mm256_cvtepu8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a1));
+ const __m256i vxa1 = _mm256_cvtepu8_epi16(va1);
+ a1 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m256i vxb01 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb01), vb_zero_point);
+
+ vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
+ vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 16));
+ const __m256i vxb23 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb23), vb_zero_point);
+
+ vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
+ vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 32));
+ const __m256i vxb45 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb45), vb_zero_point);
+
+ vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
+ vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 48));
+ const __m256i vxb67 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb67), vb_zero_point);
+
+ vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
+ vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
+
+ w = (const void*) ((const uint8_t*) w + 64);
+ k += 8 * sizeof(uint8_t);
+ }
+ p -= 2 * sizeof(void*);
+ } while (p != 0);
+
+ const __m256i vacc0x0213 = _mm256_hadd_epi32(vacc0x01, vacc0x23);
+ const __m256i vacc0x4657 = _mm256_hadd_epi32(vacc0x45, vacc0x67);
+ const __m256i vacc1x0213 = _mm256_hadd_epi32(vacc1x01, vacc1x23);
+ const __m256i vacc1x4657 = _mm256_hadd_epi32(vacc1x45, vacc1x67);
+
+ const __m256i vacc0x02461357 = _mm256_hadd_epi32(vacc0x0213, vacc0x4657);
+ const __m256i vacc1x02461357 = _mm256_hadd_epi32(vacc1x0213, vacc1x4657);
+
+ const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+ __m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
+ __m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
+
+ __m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
+ __m256 vscaled1x01234567 = _mm256_cvtepi32_ps(vacc1x01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale);
+ vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale);
+
+ vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
+ vacc1x01234567 = _mm256_cvtps_epi32(vscaled1x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
+
+ vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+
+ __m256i vout = _mm256_packus_epi16(vacc01x01234567, vacc01x01234567);
+
+ vout = _mm256_max_epu8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_min));
+ vout = _mm256_min_epu8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_max));
+
+ __m128i vout_lo = _mm256_castsi256_si128(vout);
+ __m128i vout_hi = _mm256_extracti128_si256(vout, 1);
+
+ if (nc >= 8) {
+ _mm_storel_epi64((__m128i*) c1, vout_hi);
+ _mm_storel_epi64((__m128i*) c0, vout_lo);
+
+ c1 = (uint8_t*) ((uintptr_t) c1 + cn_stride);
+ c0 = (uint8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const uint8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 8;
+ } else {
+ if (nc & 4) {
+ _mm_storeu_si32(c1, vout_hi);
+ _mm_storeu_si32(c0, vout_lo);
+
+ c1 += 4;
+ c0 += 4;
+
+ vout_lo = _mm_srli_epi64(vout_lo, 32);
+ vout_hi = _mm_srli_epi64(vout_hi, 32);
+ }
+ if (nc & 2) {
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout_hi, 0);
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0);
+
+ c1 += 2;
+ c0 += 2;
+
+ vout_lo = _mm_srli_epi32(vout_lo, 16);
+ vout_hi = _mm_srli_epi32(vout_hi, 16);
+ }
+ if (nc & 1) {
+ *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c0 = (uint8_t) _mm_extract_epi8(vout_lo, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/qu8-igemm/gen/3x8c8-minmax-fp32-avx2.c b/src/qu8-igemm/gen/3x8c8-minmax-fp32-avx2.c
new file mode 100644
index 0000000..995d2bf
--- /dev/null
+++ b/src/qu8-igemm/gen/3x8c8-minmax-fp32-avx2.c
@@ -0,0 +1,228 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/MRx8c8-avx2.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <immintrin.h>
+
+#include <xnnpack/igemm.h>
+#include <xnnpack/intrinsics-polyfill.h>
+#include <xnnpack/math.h>
+
+
+void xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2(
+ size_t mr,
+ size_t nc,
+ size_t kc,
+ size_t ks,
+ const uint8_t** restrict a,
+ const void* restrict w,
+ uint8_t* restrict c,
+ size_t cm_stride,
+ size_t cn_stride,
+ size_t a_offset,
+ const uint8_t* zero,
+ const union xnn_qu8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(mr != 0);
+ assert(mr <= 3);
+ assert(nc != 0);
+ assert(kc != 0);
+ assert(ks != 0);
+ assert(ks % (3 * sizeof(void*)) == 0);
+ assert(a_offset % sizeof(uint8_t) == 0);
+ assert(a != NULL);
+ assert(w != NULL);
+ assert(c != NULL);
+
+ kc = round_up_po2(kc, 8);
+ uint8_t* c0 = c;
+ uint8_t* c1 = (uint8_t*) ((uintptr_t) c0 + cm_stride);
+ if XNN_UNPREDICTABLE(mr < 2) {
+ c1 = c0;
+ }
+ uint8_t* c2 = (uint8_t*) ((uintptr_t) c1 + cm_stride);
+ if XNN_UNPREDICTABLE(mr <= 2) {
+ c2 = c1;
+ }
+
+ do {
+ const __m128i vbias0x0 = _mm_loadu_si32(w);
+ const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1);
+ __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1);
+ const __m128i vbias0x2 = _mm_loadu_si32((const int32_t*) w + 2);
+ const __m128i vbias0x3 = _mm_loadu_si32((const int32_t*) w + 3);
+ __m256i vacc0x23 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x2), vbias0x3, 1);
+ const __m128i vbias0x4 = _mm_loadu_si32((const int32_t*) w + 4);
+ const __m128i vbias0x5 = _mm_loadu_si32((const int32_t*) w + 5);
+ __m256i vacc0x45 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x4), vbias0x5, 1);
+ const __m128i vbias0x6 = _mm_loadu_si32((const int32_t*) w + 6);
+ const __m128i vbias0x7 = _mm_loadu_si32((const int32_t*) w + 7);
+ __m256i vacc0x67 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x6), vbias0x7, 1);
+ __m256i vacc1x01 = vacc0x01;
+ __m256i vacc1x23 = vacc0x23;
+ __m256i vacc1x45 = vacc0x45;
+ __m256i vacc1x67 = vacc0x67;
+ __m256i vacc2x01 = vacc0x01;
+ __m256i vacc2x23 = vacc0x23;
+ __m256i vacc2x45 = vacc0x45;
+ __m256i vacc2x67 = vacc0x67;
+ w = (const void*) ((const int32_t*) w + 8);
+
+ size_t p = ks;
+ const __m256i vb_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.kernel_zero_point);
+ do {
+ const uint8_t* restrict a0 = a[0];
+ if XNN_UNPREDICTABLE(a0 != zero) {
+ a0 = (const uint8_t*) ((uintptr_t) a0 + a_offset);
+ }
+ const uint8_t* restrict a1 = a[1];
+ if XNN_UNPREDICTABLE(a1 != zero) {
+ a1 = (const uint8_t*) ((uintptr_t) a1 + a_offset);
+ }
+ const uint8_t* restrict a2 = a[2];
+ if XNN_UNPREDICTABLE(a2 != zero) {
+ a2 = (const uint8_t*) ((uintptr_t) a2 + a_offset);
+ }
+ a += 3;
+
+ size_t k = 0;
+ while (k < kc) {
+ const __m128i va0 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a0));
+ const __m256i vxa0 = _mm256_cvtepu8_epi16(va0);
+ a0 += 8;
+ const __m128i va1 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a1));
+ const __m256i vxa1 = _mm256_cvtepu8_epi16(va1);
+ a1 += 8;
+ const __m128i va2 = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a2));
+ const __m256i vxa2 = _mm256_cvtepu8_epi16(va2);
+ a2 += 8;
+
+ const __m128i vb01 = _mm_load_si128((const __m128i*) w);
+ const __m256i vxb01 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb01), vb_zero_point);
+
+ vacc0x01 = _mm256_add_epi32(vacc0x01, _mm256_madd_epi16(vxa0, vxb01));
+ vacc1x01 = _mm256_add_epi32(vacc1x01, _mm256_madd_epi16(vxa1, vxb01));
+ vacc2x01 = _mm256_add_epi32(vacc2x01, _mm256_madd_epi16(vxa2, vxb01));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 16));
+ const __m256i vxb23 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb23), vb_zero_point);
+
+ vacc0x23 = _mm256_add_epi32(vacc0x23, _mm256_madd_epi16(vxa0, vxb23));
+ vacc1x23 = _mm256_add_epi32(vacc1x23, _mm256_madd_epi16(vxa1, vxb23));
+ vacc2x23 = _mm256_add_epi32(vacc2x23, _mm256_madd_epi16(vxa2, vxb23));
+ const __m128i vb45 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 32));
+ const __m256i vxb45 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb45), vb_zero_point);
+
+ vacc0x45 = _mm256_add_epi32(vacc0x45, _mm256_madd_epi16(vxa0, vxb45));
+ vacc1x45 = _mm256_add_epi32(vacc1x45, _mm256_madd_epi16(vxa1, vxb45));
+ vacc2x45 = _mm256_add_epi32(vacc2x45, _mm256_madd_epi16(vxa2, vxb45));
+ const __m128i vb67 = _mm_load_si128((const __m128i*) ((const uint8_t*) w + 48));
+ const __m256i vxb67 = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb67), vb_zero_point);
+
+ vacc0x67 = _mm256_add_epi32(vacc0x67, _mm256_madd_epi16(vxa0, vxb67));
+ vacc1x67 = _mm256_add_epi32(vacc1x67, _mm256_madd_epi16(vxa1, vxb67));
+ vacc2x67 = _mm256_add_epi32(vacc2x67, _mm256_madd_epi16(vxa2, vxb67));
+
+ w = (const void*) ((const uint8_t*) w + 64);
+ k += 8 * sizeof(uint8_t);
+ }
+ p -= 3 * sizeof(void*);
+ } while (p != 0);
+
+ const __m256i vacc0x0213 = _mm256_hadd_epi32(vacc0x01, vacc0x23);
+ const __m256i vacc0x4657 = _mm256_hadd_epi32(vacc0x45, vacc0x67);
+ const __m256i vacc1x0213 = _mm256_hadd_epi32(vacc1x01, vacc1x23);
+ const __m256i vacc1x4657 = _mm256_hadd_epi32(vacc1x45, vacc1x67);
+ const __m256i vacc2x0213 = _mm256_hadd_epi32(vacc2x01, vacc2x23);
+ const __m256i vacc2x4657 = _mm256_hadd_epi32(vacc2x45, vacc2x67);
+
+ const __m256i vacc0x02461357 = _mm256_hadd_epi32(vacc0x0213, vacc0x4657);
+ const __m256i vacc1x02461357 = _mm256_hadd_epi32(vacc1x0213, vacc1x4657);
+ const __m256i vacc2x02461357 = _mm256_hadd_epi32(vacc2x0213, vacc2x4657);
+
+ const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+ __m256i vacc0x01234567 = _mm256_permutevar8x32_epi32(vacc0x02461357, vpermute_mask);
+ __m256i vacc1x01234567 = _mm256_permutevar8x32_epi32(vacc1x02461357, vpermute_mask);
+ __m256i vacc2x01234567 = _mm256_permutevar8x32_epi32(vacc2x02461357, vpermute_mask);
+
+ __m256 vscaled0x01234567 = _mm256_cvtepi32_ps(vacc0x01234567);
+ __m256 vscaled1x01234567 = _mm256_cvtepi32_ps(vacc1x01234567);
+ __m256 vscaled2x01234567 = _mm256_cvtepi32_ps(vacc2x01234567);
+
+ const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale);
+ vscaled0x01234567 = _mm256_mul_ps(vscaled0x01234567, vscale);
+ vscaled1x01234567 = _mm256_mul_ps(vscaled1x01234567, vscale);
+ vscaled2x01234567 = _mm256_mul_ps(vscaled2x01234567, vscale);
+
+ vacc0x01234567 = _mm256_cvtps_epi32(vscaled0x01234567);
+ vacc1x01234567 = _mm256_cvtps_epi32(vscaled1x01234567);
+ vacc2x01234567 = _mm256_cvtps_epi32(vscaled2x01234567);
+
+ const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->fp32_avx2.output_zero_point);
+ __m256i vacc01x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc0x01234567, vacc1x01234567), voutput_zero_point);
+ __m256i vacc22x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc2x01234567, vacc2x01234567), voutput_zero_point);
+
+ vacc01x01234567 = _mm256_permute4x64_epi64(vacc01x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+ vacc22x01234567 = _mm256_permute4x64_epi64(vacc22x01234567, _MM_SHUFFLE(3, 1, 2, 0));
+
+ __m256i vout = _mm256_packus_epi16(vacc01x01234567, vacc22x01234567);
+
+ vout = _mm256_max_epu8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_min));
+ vout = _mm256_min_epu8(vout, _mm256_load_si256((const __m256i*) params->fp32_avx2.output_max));
+
+ __m128i vout_lo = _mm256_castsi256_si128(vout);
+ __m128i vout_hi = _mm256_extracti128_si256(vout, 1);
+
+ if (nc >= 8) {
+ _mm_storeh_pi((__m64*) c2, _mm_castsi128_ps(vout_lo));
+ _mm_storel_epi64((__m128i*) c1, vout_hi);
+ _mm_storel_epi64((__m128i*) c0, vout_lo);
+
+ c2 = (uint8_t*) ((uintptr_t) c2 + cn_stride);
+ c1 = (uint8_t*) ((uintptr_t) c1 + cn_stride);
+ c0 = (uint8_t*) ((uintptr_t) c0 + cn_stride);
+
+ a = (const uint8_t**restrict) ((uintptr_t) a - ks);
+
+ nc -= 8;
+ } else {
+ if (nc & 4) {
+ *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout_lo, 2);
+ _mm_storeu_si32(c1, vout_hi);
+ _mm_storeu_si32(c0, vout_lo);
+
+ c2 += 4;
+ c1 += 4;
+ c0 += 4;
+
+ vout_lo = _mm_srli_epi64(vout_lo, 32);
+ vout_hi = _mm_srli_epi64(vout_hi, 32);
+ }
+ if (nc & 2) {
+ *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout_lo, 4);
+ *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout_hi, 0);
+ *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0);
+
+ c2 += 2;
+ c1 += 2;
+ c0 += 2;
+
+ vout_lo = _mm_srli_epi32(vout_lo, 16);
+ vout_hi = _mm_srli_epi32(vout_hi, 16);
+ }
+ if (nc & 1) {
+ *c2 = (uint8_t) _mm_extract_epi8(vout_lo, 8);
+ *c1 = (uint8_t) _mm_extract_epi8(vout_hi, 0);
+ *c0 = (uint8_t) _mm_extract_epi8(vout_lo, 0);
+ }
+
+ nc = 0;
+ }
+ } while (nc != 0);
+}
diff --git a/src/xnnpack/gemm.h b/src/xnnpack/gemm.h
index e789e10..f46aee6 100644
--- a/src/xnnpack/gemm.h
+++ b/src/xnnpack/gemm.h
@@ -582,6 +582,10 @@
DECLARE_QU8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qu8_gemm_minmax_fp32_ukernel_2x4c8__xop_ld128)
DECLARE_QU8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qu8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128)
+DECLARE_QU8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2)
+DECLARE_QU8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2)
+DECLARE_QU8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2)
+
DECLARE_QU8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qu8_gemm_minmax_gemmlowp_ukernel_2x2__scalar)
diff --git a/src/xnnpack/igemm.h b/src/xnnpack/igemm.h
index 4c85f57..39afb40 100644
--- a/src/xnnpack/igemm.h
+++ b/src/xnnpack/igemm.h
@@ -391,6 +391,10 @@
DECLARE_QU8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qu8_igemm_minmax_fp32_ukernel_2x4c8__xop_ld128)
DECLARE_QU8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qu8_igemm_minmax_fp32_ukernel_3x4c8__xop_ld128)
+DECLARE_QU8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2)
+DECLARE_QU8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2)
+DECLARE_QU8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2)
+
DECLARE_QU8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qu8_igemm_minmax_gemmlowp_ukernel_2x2__scalar)
diff --git a/src/xnnpack/params-init.h b/src/xnnpack/params-init.h
index 4b9709d..3f87fef 100644
--- a/src/xnnpack/params-init.h
+++ b/src/xnnpack/params-init.h
@@ -41,6 +41,14 @@
uint8_t output_zero_point,
uint8_t output_min,
uint8_t output_max);
+
+XNN_INTERNAL void xnn_init_qu8_conv_minmax_fp32_avx2_params(
+ union xnn_qu8_conv_minmax_params params[XNN_MIN_ELEMENTS(1)],
+ uint8_t kernel_zero_point,
+ float scale,
+ uint8_t output_zero_point,
+ uint8_t output_min,
+ uint8_t output_max);
#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
#if XNN_ARCH_ARM || XNN_ARCH_ARM64
diff --git a/src/xnnpack/params.h b/src/xnnpack/params.h
index 0463315..bed5921 100644
--- a/src/xnnpack/params.h
+++ b/src/xnnpack/params.h
@@ -281,6 +281,13 @@
XNN_ALIGN(16) uint8_t output_min[16];
XNN_ALIGN(16) uint8_t output_max[16];
} fp32_sse2;
+ struct {
+ XNN_ALIGN(32) int16_t kernel_zero_point[16];
+ XNN_ALIGN(32) float scale[8];
+ XNN_ALIGN(32) int16_t output_zero_point[8];
+ XNN_ALIGN(32) uint8_t output_min[32];
+ XNN_ALIGN(32) uint8_t output_max[32];
+ } fp32_avx2;
#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
};
diff --git a/test/qu8-gemm-minmax-fp32.cc b/test/qu8-gemm-minmax-fp32.cc
index 90d62e7..9507af1 100644
--- a/test/qu8-gemm-minmax-fp32.cc
+++ b/test/qu8-gemm-minmax-fp32.cc
@@ -28300,3 +28300,1518 @@
}
}
#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, k_eq_8) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(8)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(8)
+ .cn_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(8)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, k_lt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, k_gt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, k_div_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, n_gt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, n_gt_8_strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, n_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, n_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, n_div_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, n_div_8_strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, n_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, n_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(11)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, qmin) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, qmax) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, strided_cm) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(8)
+ .cm_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, no_a_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .a_zero_point(0)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, no_b_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .b_zero_point(0)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_1X8C8__AVX2, no_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .a_zero_point(0)
+ .b_zero_point(0)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, k_eq_8) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(8)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(8)
+ .cn_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(8)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, k_lt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, k_gt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, k_div_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, n_gt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, n_gt_8_strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, n_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, n_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, n_div_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, n_div_8_strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, n_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, n_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(11)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, qmin) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, qmax) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, strided_cm) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(8)
+ .cm_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, no_a_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .a_zero_point(0)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, no_b_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .b_zero_point(0)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_2X8C8__AVX2, no_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .a_zero_point(0)
+ .b_zero_point(0)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, k_eq_8) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(8)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(8)
+ .cn_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, k_eq_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(8)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, k_lt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, k_lt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, k_gt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, k_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, k_div_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, k_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, n_gt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, n_gt_8_strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, n_gt_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, n_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, n_div_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, n_div_8_strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, n_div_8_strided_a) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, n_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(11)
+ .iterations(1)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, qmin) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, qmax) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, strided_cm) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(8)
+ .cm_stride(11)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, no_a_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .a_zero_point(0)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, no_b_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .b_zero_point(0)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_GEMM_MINMAX_FP32_3X8C8__AVX2, no_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .a_zero_point(0)
+ .b_zero_point(0)
+ .Test(xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
diff --git a/test/qu8-gemm-minmax-fp32.yaml b/test/qu8-gemm-minmax-fp32.yaml
index 416aabe..64a9400 100644
--- a/test/qu8-gemm-minmax-fp32.yaml
+++ b/test/qu8-gemm-minmax-fp32.yaml
@@ -171,3 +171,12 @@
- name: xnn_qu8_gemm_minmax_fp32_ukernel_3x4c8__xop_ld128
init: xnn_init_qu8_conv_minmax_fp32_sse2_params
k-block: 8
+- name: xnn_qu8_gemm_minmax_fp32_ukernel_1x8c8__avx2
+ init: xnn_init_qu8_conv_minmax_fp32_avx2_params
+ k-block: 8
+- name: xnn_qu8_gemm_minmax_fp32_ukernel_2x8c8__avx2
+ init: xnn_init_qu8_conv_minmax_fp32_avx2_params
+ k-block: 8
+- name: xnn_qu8_gemm_minmax_fp32_ukernel_3x8c8__avx2
+ init: xnn_init_qu8_conv_minmax_fp32_avx2_params
+ k-block: 8
diff --git a/test/qu8-igemm-minmax-fp32.cc b/test/qu8-igemm-minmax-fp32.cc
index 3944789..8d773a4 100644
--- a/test/qu8-igemm-minmax-fp32.cc
+++ b/test/qu8-igemm-minmax-fp32.cc
@@ -28972,3 +28972,1554 @@
}
}
#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, k_eq_8) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(8)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(8)
+ .cn_stride(11)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(8)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, k_lt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, k_gt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, k_div_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, n_gt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, n_gt_8_strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, n_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, n_div_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, n_div_8_strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(n)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, n_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, small_kernel) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, small_kernel_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .ks(3)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, n_gt_8_small_kernel) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, n_div_8_small_kernel) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 1; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(11)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, a_offset) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .a_offset(43)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, zero) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t mz = 0; mz < 1; mz++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .a_offset(43)
+ .zero_index(mz)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, qmin) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, qmax) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, strided_cm) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(8)
+ .cm_stride(11)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, no_a_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .a_zero_point(0)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, no_b_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .b_zero_point(0)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_1X8C8__AVX2, no_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(1)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(1)
+ .n(8)
+ .k(k)
+ .a_zero_point(0)
+ .b_zero_point(0)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, k_eq_8) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(8)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(8)
+ .cn_stride(11)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(8)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, k_lt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, k_gt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, k_div_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, n_gt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, n_gt_8_strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, n_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, n_div_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, n_div_8_strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(n)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, n_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, small_kernel) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, small_kernel_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .ks(3)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, n_gt_8_small_kernel) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, n_div_8_small_kernel) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 2; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(11)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, a_offset) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .a_offset(83)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, zero) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t mz = 0; mz < 2; mz++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .a_offset(83)
+ .zero_index(mz)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, qmin) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, qmax) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, strided_cm) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(8)
+ .cm_stride(11)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, no_a_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .a_zero_point(0)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, no_b_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .b_zero_point(0)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_2X8C8__AVX2, no_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(2)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(2)
+ .n(8)
+ .k(k)
+ .a_zero_point(0)
+ .b_zero_point(0)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
+
+
+#if XNN_ARCH_X86 || XNN_ARCH_X86_64
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, k_eq_8) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(8)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(8)
+ .cn_stride(11)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, k_eq_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, k_eq_8_subtile_m) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(8)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, k_eq_8_subtile_n) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, k_lt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, k_lt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, k_gt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, k_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, k_div_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, k_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, n_gt_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, n_gt_8_strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, n_gt_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, n_div_8) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, n_div_8_strided_cn) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(n)
+ .k(k)
+ .cn_stride(11)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, n_div_8_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, small_kernel) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, small_kernel_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .ks(3)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, n_gt_8_small_kernel) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 9; n < 16; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, n_div_8_small_kernel) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t n = 16; n <= 24; n += 8) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, strided_cm_subtile) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 3; m++) {
+ for (uint32_t n = 1; n <= 8; n++) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(11)
+ .iterations(1)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, a_offset) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .a_offset(127)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, zero) {
+ TEST_REQUIRES_X86_AVX2;
+ for (uint32_t mz = 0; mz < 3; mz++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .ks(3)
+ .a_offset(127)
+ .zero_index(mz)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, qmin) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, qmax) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, strided_cm) {
+ TEST_REQUIRES_X86_AVX2;
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(8)
+ .cm_stride(11)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, no_a_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .a_zero_point(0)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, no_b_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .b_zero_point(0)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+
+ TEST(QU8_IGEMM_MINMAX_FP32_3X8C8__AVX2, no_zero_point) {
+ TEST_REQUIRES_X86_AVX2;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(3)
+ .nr(8)
+ .kr(8)
+ .sr(1)
+ .m(3)
+ .n(8)
+ .k(k)
+ .a_zero_point(0)
+ .b_zero_point(0)
+ .Test(xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2, xnn_init_qu8_conv_minmax_fp32_avx2_params, xnn_init_qu8_requantization_fp32_params, xnn_qu8_requantize_fp32);
+ }
+ }
+#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
diff --git a/test/qu8-igemm-minmax-fp32.yaml b/test/qu8-igemm-minmax-fp32.yaml
index 4f1ca15..da8f89c 100644
--- a/test/qu8-igemm-minmax-fp32.yaml
+++ b/test/qu8-igemm-minmax-fp32.yaml
@@ -171,3 +171,12 @@
- name: xnn_qu8_igemm_minmax_fp32_ukernel_3x4c8__xop_ld128
init: xnn_init_qu8_conv_minmax_fp32_sse2_params
k-block: 8
+- name: xnn_qu8_igemm_minmax_fp32_ukernel_1x8c8__avx2
+ init: xnn_init_qu8_conv_minmax_fp32_avx2_params
+ k-block: 8
+- name: xnn_qu8_igemm_minmax_fp32_ukernel_2x8c8__avx2
+ init: xnn_init_qu8_conv_minmax_fp32_avx2_params
+ k-block: 8
+- name: xnn_qu8_igemm_minmax_fp32_ukernel_3x8c8__avx2
+ init: xnn_init_qu8_conv_minmax_fp32_avx2_params
+ k-block: 8